Add support for beta channel

README.md

@@ -11,8 +11,8 @@ so the evaluation is *pure* and no need to have network (but [nixpkgs-mozilla][m
 It also works well with [Nix Flakes](https://nixos.wiki/wiki/Flakes).
 
 - The toolchain hashes are auto-updated daily using GitHub Actions.
-- Current oldest supported version is stable 1.29.0 and nightly 2018-09-13
- (which is randomly chosen).
+- Current oldest supported version is stable 1.29.0 and beta/nightly 2018-09-13
+ (which are randomly chosen).
 
 ## Use as a classic Nix overlay
 
@@ -108,6 +108,14 @@ Here's an example of using it in nixos configuration.
  # ... other versions.
  };
 
+ beta = {
+ # The latest beta toolchain.
+ latest = { /* toolchain */ };
+ "2021-01-01" = { /* toolchain */ };
+ "2020-12-30" = { /* toolchain */ };
+ # ... other versions.
+ };
+
  nightly = {
  # The latest nightly toolchain.
  latest = { /* toolchain */ };
@@ -130,10 +138,12 @@ Here's an example of using it in nixos configuration.
 
 Some examples (assume `nixpkgs` had the overlay applied):
 
-- Latest stable rust with all default components:
- `nixpkgs.rust-bin.stable.latest.rust`
-- Latest nightly rust with all default components:
- `nixpkgs.rust-bin.nightly.latest.rust`
+- Latest stable/beta/nightly rust with all default components:
+ `nixpkgs.rust-bin.{stable,beta,nightly}.latest.rust`
+- A specific version of stable rust:
+ `nixpkgs.rust-bin.stable."1.48.0".rust`
+- A specific date of beta rust:
+ `nixpkgs.rust-bin.nightly."2021-01-01".rust`
 - A specific version of stable rust:
  `nixpkgs.rust-bin.stable."1.48.0".rust`
 - A specific date of nightly rust:

fetch.py

@@ -17,7 +17,7 @@
 SYNC_MAX_FETCH = 5
 
 MIN_STABLE_VERSION = '1.29.0'
-MIN_NIGHTLY_DATE = datetime.date.fromisoformat('2018-09-13')
+MIN_BETA_DATE = MIN_NIGHTLY_DATE = datetime.date.fromisoformat('2018-09-13')
 
 DIST_ROOT = 'https://static.rust-lang.org/dist'
 NIX_KEYWORDS = {'', 'if', 'then', 'else', 'assert', 'with', 'let', 'in', 'rec', 'inherit', 'or'}
@@ -86,19 +86,24 @@ def compress_renames(renames: dict) -> int:
  f.write(']\n')
  return idx
 
-def retry_with(f):
+def fetch_url(url: str, params=None, allow_not_found=False):
  i = 0
  while True:
+ resp = None
  try:
- return f()
+ resp = requests.get(url, params=params)
+ if resp.status_code == 404 and allow_not_found:
+ return None
+ resp.raise_for_status()
+ return resp
  except requests.exceptions.RequestException as e:
  i += 1
- if i >= MAX_TRIES:
+ if (resp is not None and resp.status_code == 404) or i >= MAX_TRIES:
  raise
  print(e)
  time.sleep(RETRY_DELAY)
 
-def translate_dump_manifest(manifest: str, f, nightly=False):
+def translate_dump_manifest(channel: str, manifest: str, f):
  manifest = toml.loads(manifest)
  date = manifest['date']
  rustc_version = manifest['pkg']['rustc']['version'].split()[0]
@@ -148,20 +153,25 @@ def translate_dump_manifest(manifest: str, f, nightly=False):
  f.write('};')
  f.write('}\n')
 
-def fetch_stable_manifest(version: str, out_path: Path):
+def fetch_manifest(channel: str, version: str, out_path: Path):
  out_path.parent.mkdir(parents=True, exist_ok=True)
  tmp_path = out_path.with_suffix('.tmp')
- print(f'Fetching stable {version}')
- manifest = retry_with(lambda: requests.get(f'{DIST_ROOT}/channel-rust-{version}.toml'))
- manifest.raise_for_status()
+ print(f'Fetching {channel} {version}')
+ if channel == 'stable':
+ url = f'{DIST_ROOT}/channel-rust-{version}.toml'
+ else:
+ url = f'{DIST_ROOT}/{version}/channel-rust-{channel}.toml'
+ manifest = fetch_url(url, allow_not_found=channel != 'stable')
+ if manifest is None:
+ print('Not found, skipped')
+ return
  manifest = manifest.text
  MANIFEST_TMP_PATH.write_text(manifest)
  with open(tmp_path, 'w') as fout:
- translate_dump_manifest(manifest, fout)
+ translate_dump_manifest(channel, manifest, fout)
  tmp_path.rename(out_path)
 
-def update_stable_index():
- dir = Path('manifests/stable')
+def update_stable_index(dir=Path('manifests/stable')):
  versions = sorted(
  (file.stem for file in dir.iterdir() if file.stem != 'default' and file.suffix == '.nix'),
  key=parse_version,
@@ -173,6 +183,19 @@ def update_stable_index():
  f.write(f' latest = {escape_nix_string(versions[-1])};\n')
  f.write('}\n')
 
+def update_beta_index():
+ update_nightly_index(dir=Path('manifests/beta'))
+
+def update_nightly_index(dir=Path('manifests/nightly')):
+ dates = sorted(file.stem for file in dir.rglob('*.nix') if file.stem != 'default')
+ with open(str(dir / 'default.nix'), 'w') as f:
+ f.write('{\n')
+ for date in dates:
+ year = date.split('-')[0]
+ f.write(f' {escape_nix_key(date)} = import ./{year}/{date}.nix;\n')
+ f.write(f' latest = {escape_nix_string(dates[-1])};\n')
+ f.write('}\n')
+
 def sync_stable_channel(*, stop_if_exists, max_fetch=None):
  GITHUB_RELEASES_URL = 'https://api.github.com/repos/rust-lang/rust/releases'
  PER_PAGE = 100
@@ -182,12 +205,10 @@ def sync_stable_channel(*, stop_if_exists, max_fetch=None):
  while True:
  page += 1
  print(f'Fetching release page {page}')
- release_page = retry_with(lambda: requests.get(
+ release_page = fetch_url(
  GITHUB_RELEASES_URL,
  params={'per_page': PER_PAGE, 'page': page},
- ))
- release_page.raise_for_status()
- release_page = release_page.json()
+ ).json()
  versions.extend(
  tag['tag_name']
  for tag in release_page
@@ -208,81 +229,89 @@ def sync_stable_channel(*, stop_if_exists, max_fetch=None):
  continue
  print(f'{version} is already fetched. Stopped')
  break
- fetch_stable_manifest(version, out_path)
+ fetch_manifest('stable', version, out_path)
  processed += 1
  assert max_fetch is None or processed <= max_fetch, 'Too many versions'
  update_stable_index()
 
-def fetch_nightly_manifest(date: str, out_path: Path):
- out_path.parent.mkdir(parents=True, exist_ok=True)
- tmp_path = out_path.with_suffix('.tmp')
- print(f'Fetching nightly {date}')
- manifest = retry_with(lambda: requests.get(f'{DIST_ROOT}/{date}/channel-rust-nightly.toml'))
- if manifest.status_code == 404:
- print(f'Not found, skipped')
- return
- manifest.raise_for_status()
- manifest = manifest.text
- MANIFEST_TMP_PATH.write_text(manifest)
- with open(tmp_path, 'w') as fout:
- translate_dump_manifest(manifest, fout, nightly=True)
- tmp_path.rename(out_path)
+def sync_beta_channel(*, stop_if_exists, max_fetch=None):
+ # Fetch the global nightly manifest to retrive the latest nightly version.
+ print('Fetching latest beta version')
+ manifest = fetch_url(f'{DIST_ROOT}/channel-rust-beta.toml').text
+ date = datetime.date.fromisoformat(toml.loads(manifest)['date'])
+ print(f'The latest beta version is {date}')
+
+ processed = 0
+ date += datetime.timedelta(days=1)
+ while date > MIN_BETA_DATE:
+ date -= datetime.timedelta(days=1)
+ date_str = date.isoformat()
+ out_path = Path(f'manifests/beta/{date.year}/{date_str}.nix')
+ if out_path.exists():
+ if not stop_if_exists:
+ continue
+ print(f'{date_str} is already fetched. Stopped')
+ break
+ fetch_manifest('beta', date_str, out_path)
+ processed += 1
+ assert max_fetch is None or processed <= max_fetch, 'Too many versions'
+ update_beta_index()
 
 def sync_nightly_channel(*, stop_if_exists, max_fetch=None):
  # Fetch the global nightly manifest to retrive the latest nightly version.
  print('Fetching latest nightly version')
- manifest = retry_with(lambda: requests.get(f'{DIST_ROOT}/channel-rust-nightly.toml'))
- manifest.raise_for_status()
- date = datetime.date.fromisoformat(toml.loads(manifest.text)['date'])
+ manifest = fetch_url(f'{DIST_ROOT}/channel-rust-nightly.toml').text
+ date = datetime.date.fromisoformat(toml.loads(manifest)['date'])
  print(f'The latest nightly version is {date}')
 
  processed = 0
  date += datetime.timedelta(days=1)
  while date > MIN_NIGHTLY_DATE:
  date -= datetime.timedelta(days=1)
- out_path = Path(f'manifests/nightly/{date.year}/{date.isoformat()}.nix')
+ date_str = date.isoformat()
+ out_path = Path(f'manifests/nightly/{date.year}/{date_str}.nix')
  if out_path.exists():
  if not stop_if_exists:
  continue
- print(f'{date} is already fetched. Stopped')
+ print(f'{date_str} is already fetched. Stopped')
  break
- fetch_nightly_manifest(date.isoformat(), out_path)
+ fetch_manifest('nightly', date_str, out_path)
  processed += 1
  assert max_fetch is None or processed <= max_fetch, 'Too many versions'
  update_nightly_index()
 
-def update_nightly_index():
- dir = Path('manifests/nightly')
- dates = sorted(file.stem for file in dir.rglob('*.nix') if file.stem != 'default')
- with open(str(dir / 'default.nix'), 'w') as f:
- f.write('{\n')
- for date in dates:
- year = date.split('-')[0]
- f.write(f' {escape_nix_key(date)} = import ./{year}/{date}.nix;\n')
- f.write(f' latest = {escape_nix_string(dates[-1])};\n')
- f.write('}\n')
-
 def main():
  args = sys.argv[1:]
  if len(args) == 0:
  print('Synchronizing stable channels')
  sync_stable_channel(stop_if_exists=True, max_fetch=SYNC_MAX_FETCH)
- print('Synchronizing nightly channels')
+ print('\nSynchronizing beta channels')
+ sync_beta_channel(stop_if_exists=True, max_fetch=SYNC_MAX_FETCH)
+ print('\nSynchronizing nightly channels')
  sync_nightly_channel(stop_if_exists=True, max_fetch=SYNC_MAX_FETCH)
  elif len(args) == 2 and args[0] == 'stable':
  if args[1] == 'all':
  sync_stable_channel(stop_if_exists=False)
  else:
  version = args[1]
  assert RE_STABLE_VERSION.match(version), 'Invalid version'
- fetch_stable_manifest(version, Path(f'manifests/stable/{version}.nix'))
+ fetch_manifest('stable', version, Path(f'manifests/stable/{version}.nix'))
  update_stable_index()
+ elif len(args) == 2 and args[0] == 'beta':
+ if args[1] == 'all':
+ sync_beta_channel(stop_if_exists=False)
+ else:
+ date = datetime.date.fromisoformat(args[1])
+ date_str = date.isoformat()
+ fetch_manifest('beta', date_str, Path(f'manifests/beta/{date.year}/{date_str}.nix'))
+ update_beta_index()
  elif len(args) == 2 and args[0] == 'nightly':
  if args[1] == 'all':
  sync_nightly_channel(stop_if_exists=False)
  else:
  date = datetime.date.fromisoformat(args[1])
- fetch_nightly_manifest(date, Path(f'manifests/nightly/{date.year}/{date.isoformat()}.nix'))
+ date_str = date.isoformat()
+ fetch_manifest('nightly', date_str, Path(f'manifests/nightly/{date.year}/{date_str}.nix'))
  update_nightly_index()
  else:
  print('''

flake.nix

@@ -53,7 +53,7 @@
 
  checks = let
  inherit (pkgs) rust-bin rustChannelOf;
- inherit (pkgs.rust-bin) fromRustupToolchain fromRustupToolchainFile stable nightly;
+ inherit (pkgs.rust-bin) fromRustupToolchain fromRustupToolchainFile stable beta nightly;
 
  rustTarget = pkgs.rust.toRustTarget pkgs.hostPlatform;
 
@@ -71,6 +71,7 @@
  url-kind-0 = assertUrl stable."1.47.0".cargo "https://static.rust-lang.org/dist/2020-10-08/cargo-0.48.0-${rustTarget}.tar.xz";
  url-kind-1 = assertUrl stable."1.34.2".llvm-tools-preview "https://static.rust-lang.org/dist/2019-05-14/llvm-tools-1.34.2%20(6c2484dc3%202019-05-13)-${rustTarget}.tar.xz";
  url-kind-nightly = assertUrl nightly."2021-01-01".rustc "https://static.rust-lang.org/dist/2021-01-01/rustc-nightly-${rustTarget}.tar.xz";
+ url-kind-beta = assertUrl beta."2021-01-01".rustc "https://static.rust-lang.org/dist/2021-01-01/rustc-beta-${rustTarget}.tar.xz";
  url-fix = assertUrl nightly."2019-01-10".rustc "https://static.rust-lang.org/dist/2019-01-10/rustc-nightly-${rustTarget}.tar.xz";
 
  rename-available = assertEq stable."1.48.0".rustfmt stable."1.48.0".rustfmt-preview;
@@ -80,17 +81,22 @@
  };
 
  latest-stable = assertEq pkgs.latest.rustChannels.stable.rust stable.latest.rust;
+ latest-beta = assertEq pkgs.latest.rustChannels.beta.rust beta.latest.rust;
  latest-nightly = assertEq pkgs.latest.rustChannels.nightly.rust nightly.latest.rust;
 
  rust-channel-of-stable = assertEq (rustChannelOf { channel = "stable"; }).rust stable.latest.rust;
+ rust-channel-of-beta = assertEq (rustChannelOf { channel = "beta"; }).rust beta.latest.rust;
  rust-channel-of-nightly = assertEq (rustChannelOf { channel = "nightly"; }).rust nightly.latest.rust;
  rust-channel-of-version = assertEq (rustChannelOf { channel = "1.48.0"; }).rust stable."1.48.0".rust;
  rust-channel-of-nightly-date = assertEq (rustChannelOf { channel = "nightly"; date = "2021-01-01"; }).rust nightly."2021-01-01".rust;
+ rust-channel-of-beta-date = assertEq (rustChannelOf { channel = "beta"; date = "2021-01-01"; }).rust beta."2021-01-01".rust;
 
  rustup-toolchain-stable = assertEq (fromRustupToolchain { channel = "stable"; }) stable.latest.rust;
+ rustup-toolchain-beta = assertEq (fromRustupToolchain { channel = "beta"; }) beta.latest.rust;
  rustup-toolchain-nightly = assertEq (fromRustupToolchain { channel = "nightly"; }) nightly.latest.rust;
  rustup-toolchain-version = assertEq (fromRustupToolchain { channel = "1.48.0"; }) stable."1.48.0".rust;
  rustup-toolchain-nightly-date = assertEq (fromRustupToolchain { channel = "nightly-2021-01-01"; }) nightly."2021-01-01".rust;
+ rustup-toolchain-beta-date = assertEq (fromRustupToolchain { channel = "beta-2021-01-01"; }) beta."2021-01-01".rust;
  rustup-toolchain-customization = assertEq
  (fromRustupToolchain {
  channel = "1.48.0";

manifest.nix

@@ -42,7 +42,7 @@ let
 
  # Uncompress the compressed manifest to the original one
  # (not complete but has enough information to make up the toolchain).
- uncompressManifest = nightly: version: {
+ uncompressManifest = channel: version: {
  v, # rustc version
  d, # date
  r, # rename index
@@ -61,9 +61,9 @@ let
  pkgNameStripped = removeSuffix "-preview" pkgName;
  targetTail = if targetIdx == "_" then "" else "-" + target;
  urlVersion =
- if u != null then u # Use specified url version if exists.
- else if nightly then "nightly" # Otherwise, for nightly channel, default to be "nightly".
- else v; # For stable channel, default to be rustc version.
+ if u != null then u  # Use specified url version if exists.
+ else if channel == "stable" then v # For stable channel, default to be rustc version.
+ else channel;  # Otherwise, for beta/nightly channel, default to be "beta"/"nightly".
  in {
  name = target;
  value = {
@@ -74,8 +74,8 @@ let
  }) (removeAttrs manifest ["v" "d" "r"]);
  };
 
- uncompressManifestSet = nightly: set: let
- ret = mapAttrs (uncompressManifest nightly) (removeAttrs set ["latest"]);
+ uncompressManifestSet = channel: set: let
+ ret = mapAttrs (uncompressManifest channel) (removeAttrs set ["latest"]);
  in ret // { latest = ret.${set.latest}; };
 
 in {
@@ -86,8 +86,9 @@ in {
 
  # For internal usage.
  manifests = {
- stable = uncompressManifestSet false (import ./manifests/stable);
- nightly = uncompressManifestSet true (import ./manifests/nightly);
+ stable = uncompressManifestSet "stable" (import ./manifests/stable);
+ beta = uncompressManifestSet "beta" (import ./manifests/beta);
+ nightly = uncompressManifestSet "nightly" (import ./manifests/nightly);
  };
  };
 }

manifests/beta/default.nix

@@ -0,0 +1,4 @@
+{
+ "2021-01-01" = import ./2021/2021-01-01.nix;
+ latest = "2021-01-01";
+}