[ENH] Milestone

examples/dataset_cifar_workflow/benchmarks/dataset/data.py

@@ -13,7 +13,7 @@
 cifar_train_set_augment = datasets.CIFAR10(root="cache", download=True,
  train=True, transform=augment_transform)
 cifar_train_set = datasets.CIFAR10(root="cache", download=True,
- train=True, transform=augment_transform)
+ train=True, transform=regular_transform)
 cifar_test_set = datasets.CIFAR10(root="cache", download=True,
  train=False, transform=regular_transform)
 
@@ -24,3 +24,8 @@ def uploader_data():
  return (Subset(cifar_train_set_augment, train_indices),
  Subset(cifar_test_set, valid_indices),
  Subset(cifar_train_set, train_indices))
+
+def user_data():
+ test_indices, order = split_dataset(torch.asarray(cifar_test_set.targets), 3000, split="user")
+
+ return Subset(cifar_test_set, test_indices)

examples/dataset_cifar_workflow/benchmarks/dataset/utils.py

@@ -4,14 +4,11 @@
 import numpy as np
 import torch
 import torchvision
-from torch.utils.data import TensorDataset
 
 torchvision.disable_beta_transforms_warning()
 from torchvision.transforms import transforms, v2
 
 
-
-
 def sample_by_labels(labels: torch.Tensor, weights, total_num):
  weights = np.asarray(weights)
 
@@ -51,17 +48,19 @@ def split_dataset(labels, size, split="uploader", order=None):
 def build_transform(size):
  augment_transform = transforms.Compose([
  transforms.Resize(size),
+ # transforms.RandomCrop(size, padding=4),
+ # transforms.RandomHorizontalFlip(),
  v2.AutoAugment(),
  transforms.ToTensor(),
- transforms.Normalize(mean=[0.485, 0.456, 0.406],
- std=[0.229, 0.224, 0.225]),
+ transforms.Normalize(mean=(0.4914, 0.4822, 0.4465),
+ std=(0.2023, 0.1994, 0.2010)),
  ])
 
  regular_transform = transforms.Compose([
  transforms.Resize(size),
  transforms.ToTensor(),
- transforms.Normalize(mean=[0.485, 0.456, 0.406],
- std=[0.229, 0.224, 0.225]),
+ transforms.Normalize(mean=(0.4914, 0.4822, 0.4465),
+ std=(0.2023, 0.1994, 0.2010)),
  ])
 
  return augment_transform, regular_transform

examples/dataset_cifar_workflow/benchmarks/models/conv/__init__.py

@@ -22,5 +22,8 @@ def fit(self, X: np.ndarray, y: np.ndarray):
  def predict(self, X: np.ndarray) -> np.ndarray:
  return self.model(torch.asarray(X, dtype=torch.float32, device=self.device))
 
+ def __call__(self, *args, **kwargs):
+ self.predict(*args, **kwargs)
+
  def finetune(self, X: np.ndarray, y: np.ndarray):
  raise NotImplementedError()

examples/dataset_cifar_workflow/benchmarks/models/config.yaml → examples/dataset_cifar_workflow/benchmarks/models/learnware.yaml

@@ -3,6 +3,6 @@ model:
  kwargs: {}
 stat_specifications:
  - module_path: learnware.specification
- class_name: RKMEImageStatSpecification
+ class_name: RKMEImageSpecification
  file_name: spec.json
  kwargs: {}

examples/dataset_cifar_workflow/benchmarks/utils.py

@@ -4,14 +4,13 @@
 
 import numpy as np
 import torch
-import tqdm
 from torch import optim, nn
 from torch.utils.data import DataLoader, Dataset
 
 from learnware.client import LearnwareClient
 from learnware.learnware import Learnware
 from learnware.specification import generate_rkme_image_spec, RKMEImageSpecification
-from .dataset import uploader_data
+from .dataset import uploader_data, user_data
 from .models.conv import ConvModel
 from learnware.market import LearnwareMarket
 from learnware.utils import choose_device
@@ -20,18 +19,20 @@
 def evaluate(model, evaluate_set: Dataset, device=None):
  device = choose_device(0) if device is None else device
 
- if isinstance(model, Learnware):
- # duck-type
- model.__call__ = model.predict
  if isinstance(model, nn.Module):
  model.eval()
+ mapping = lambda m, x: m(x)
+ elif isinstance(model, Learnware):
+ mapping = lambda m, x: m.predict(x)
+ else:
+ raise Exception("not support model type", model)
 
  criterion = nn.CrossEntropyLoss(reduction="sum")
  total, correct, loss = 0, 0, 0.0
  dataloader = DataLoader(evaluate_set, batch_size=512, shuffle=True)
  for i, (X, y) in enumerate(dataloader):
  X, y = X.to(device), y.to(device)
- out = model(X)
+ out = mapping(model, X)
  loss += criterion(out, y)
 
  _, predicted = torch.max(out.data, 1)
@@ -48,30 +49,42 @@ def evaluate(model, evaluate_set: Dataset, device=None):
 
 
 def build_learnware(name: str, market: LearnwareMarket, model_name="conv",
- out_classes=10, epochs=35, batch_size=1024, device=None):
+ out_classes=10, epochs=200, batch_size=2048, device=None):
  device = choose_device(0) if device is None else device
 
  if name == "cifar10":
  train_set, valid_set, spec_set = uploader_data()
  else:
  raise Exception("Not support", name)
 
+ cache_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'cache', 'learnware'))
+ if os.path.exists(cache_dir):
+ rmtree(cache_dir)
+ os.makedirs(cache_dir, exist_ok=True)
+
  channel = train_set[0][0].shape[0]
  image_size = train_set[0][0].shape[1], train_set[0][0].shape[2]
 
  model = ConvModel(channel=channel, im_size=image_size,
  n_random_features=out_classes).to(device)
+ # if device.type == 'cuda':
+ # model = nn.DataParallel(model)
+ # model.benchmark = True
+
  model.train()
 
  # SGD optimizer with learning rate 1e-2
- optimizer = optim.SGD(model.parameters(), lr=1e-2, momentum=0.9)
-
+ optimizer = optim.SGD(model.parameters(), lr=5e-2, momentum=0.9)
+ # Scheduler TODO: Use this
+ # scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=20)
  # mean-squared error loss
  criterion = nn.CrossEntropyLoss()
  # Prepare DataLoader
  dataloader = DataLoader(train_set, batch_size=batch_size, shuffle=True)
+ # valid loss
+ best_loss = 100000 # initially
  # Optimizing...
- for epoch in tqdm.tqdm(range(epochs), total=epochs):
+ for epoch in range(epochs):
  running_loss = []
  for i, (X, y) in enumerate(dataloader):
  X, y = X.to(device=device), y.to(device=device)
@@ -82,41 +95,48 @@ def build_learnware(name: str, market: LearnwareMarket, model_name="conv",
  optimizer.step()
  running_loss.append(loss.item())
 
+ valid_loss, valid_acc = evaluate(model, train_set, device=device)
+ if valid_loss < best_loss:
+ best_loss = valid_loss
+ if isinstance(model, nn.DataParallel):
+ model_to_save = model.module
+ else:
+ model_to_save = model
+ torch.save(model_to_save.state_dict(), os.path.join(cache_dir, "model.pth"))
+ print("Epoch: {}, Valid Best Accuracy: {:.3f}% ({:.3f})".format(epoch+1, valid_acc, valid_loss))
+
  if (epoch + 1) % 5 == 0:
- valid_loss, valid_acc = evaluate(model, train_set, device=device)
  print('Epoch: {}, Train Average Loss: {:.3f}, Valid Average Loss: {:.3f}'.format(
  epoch+1, np.mean(running_loss), valid_loss))
 
- train_loss, train_acc = evaluate(model, train_set, device=device)
- print("Train Loss: {:.3e}\tTrain Accuracy: {:.3e}".format(train_loss, train_acc))
+ # scheduler.step()
 
  # build specification
  loader = DataLoader(spec_set, batch_size=3000, shuffle=True)
  sampled_X, _ = next(iter(loader))
  spec = generate_rkme_image_spec(sampled_X)
 
  # add to market
- cache_dir = os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..', 'cache', 'learnware'))
- if os.path.exists(cache_dir):
- rmtree(cache_dir)
- os.makedirs(cache_dir, exist_ok=True)
- model_dir = os.path.abspath(os.path.join(__file__, "models"))
+ model_dir = os.path.abspath(os.path.join(__file__, "..", "models"))
  spec.save(os.path.join(cache_dir, "spec.json"))
 
  zip_file = os.path.join(cache_dir, "learnware.zip")
  # zip -q -r -j zip_file dir_path
  with zipfile.ZipFile(zip_file, "w") as zip_obj:
  for foldername, subfolders, filenames in os.walk(os.path.join(model_dir, model_name)):
  for filename in filenames:
+ if filename.endswith(".pyc"):
+ continue
  file_path = os.path.join(foldername, filename)
  zip_info = zipfile.ZipInfo(filename)
  zip_info.compress_type = zipfile.ZIP_STORED
  with open(file_path, "rb") as file:
  zip_obj.writestr(zip_info, file.read())
 
- for filename, filepath in zip(["spec.json", "config.yaml"],
+ for filename, file_path in zip(["spec.json", "model.pth", "learnware.yaml"],
  [os.path.join(cache_dir, "spec.json"),
- os.path.join(model_dir, "config.yaml")]):
+ os.path.join(cache_dir, "model.pth"),
+ os.path.join(model_dir, "learnware.yaml")]):
  zip_info = zipfile.ZipInfo(filename)
  zip_info.compress_type = zipfile.ZIP_STORED
  with open(file_path, "rb") as file:
@@ -125,34 +145,34 @@ def build_learnware(name: str, market: LearnwareMarket, model_name="conv",
  market.add_learnware(zip_file, semantic_spec=LearnwareClient.create_semantic_specification(
  self=None,
  name="learnware",
- description="",
+ description="For Cifar Dataset Workflow",
  data_type="Image",
  task_type="Classification",
  library_type="PyTorch",
  scenarios=["Computer"],
- output_description={str(i): "i" for i in range(out_classes)})
+ output_description={"Dimension": out_classes, "Description": {str(i): "i" for i in range(out_classes)}})
  )
 
  return model
 
 
 def build_specification(name: str, cache_id, sampled_size=3000):
  cache_path = os.path.abspath(os.path.join(
- os.path.dirname( __file__ ), '..', '..', 'cache', "{}.json".format(cache_id)))
-
- if os.path.exists(cache_path):
- spec = RKMEImageSpecification()
- spec.load(cache_path)
- return spec
+ os.path.dirname( __file__ ), '..', 'cache', "{}.json".format(cache_id)))
 
  if name == "cifar10":
- dataset = cifar10(split="user")
+ dataset = user_data()
  else:
  raise Exception("Not support", name)
 
+ if os.path.exists(cache_path):
+ spec = RKMEImageSpecification()
+ spec.load(cache_path)
+ return spec, dataset
+
  loader = DataLoader(dataset, batch_size=sampled_size, shuffle=True)
  sampled_X, _ = next(iter(loader))
- spec = generate_rkme_image_spec(sampled_X)
+ spec = generate_rkme_image_spec(sampled_X, steps=1)
 
  spec.save(cache_path)
- return spec
+ return spec, dataset

examples/dataset_cifar_workflow/main.py

@@ -2,8 +2,9 @@
 
 import fire
 
-from examples.dataset_cifar_workflow.benchmarks.utils import build_learnware, build_specification
-from learnware.market import instantiate_learnware_market
+from benchmarks.utils import build_learnware, build_specification, evaluate
+from learnware.client import LearnwareClient
+from learnware.market import instantiate_learnware_market, BaseUserInfo
 
 PROXY_IP = "172.24.57.111"
 os.environ["HTTP_PROXY"] = "http:https://"+PROXY_IP+":7890"
@@ -18,15 +19,39 @@ def prepare_learnware(self, market_size=30, rebuild=False):
  market = instantiate_learnware_market(name="easy", market_id="dataset_cifar_workflow", rebuild=rebuild)
 
  for i in range(market_size - len(market)):
+ print("=" * 20 + "learnware {}".format(i) + "=" * 20)
  build_learnware("cifar10", market)
 
  print("Total Item:", len(market))
 
  def evaluate(self, user_size=20):
- market = instantiate_learnware_market(name="easy", market_id="dataset_cifar_workflow", rebuild=rebuild)
-
- # for i in range(user_size):
- # build_specification()
+ market = instantiate_learnware_market(name="easy", market_id="dataset_cifar_workflow", rebuild=False)
+
+ for i in range(user_size):
+ user_spec, dataset = build_specification("cifar10", i)
+
+ user_info = BaseUserInfo(semantic_spec=LearnwareClient.create_semantic_specification(
+ self=None,
+ description="For Cifar Dataset Workflow",
+ data_type="Image",
+ task_type="Classification",
+ library_type="PyTorch",
+ scenarios=["Computer"],
+ output_description={"Dimension": 10, "Description": {str(i): "i" for i in range(10)}}),
+ stat_info={"RKMEImageSpecification": user_spec})
+
+ search_result = market.search_learnware(user_info)
+ single_result = search_result.get_single_results()
+ multiple_result = search_result.get_multiple_results()
+
+ loss_list = []
+ for single_item in single_result[:3]:
+ loss, acc = evaluate(single_item.learnware, dataset)
+ loss_list.append(loss)
+
+ print(
+ f"Top1-score: {single_result[0].score}, learnware_id: {single_result[0].learnware.id}, loss: {loss_list[0]}"
+ )
 
 
 if __name__ == "__main__":

examples/dataset_cifar_workflow/run.bash

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC1090
+source ~/anaconda3/etc/profile.d/conda.sh
+conda activate dev
+
+export PYTHONPATH="${PYTHONPATH}:${HOME}/Lab/Learnware/"
+echo ${PYTHONPATH}
+token="$(date +%s)"
+mkdir -p "./log"
+echo "The output is redirected to log/${token}.log with token ${token}"
+
+# shellcheck disable=SC2086
+nohup python -u main.py prepare_learnware --rebuild=True > "./log/${token}.log" 2>&1 &
+echo "With PID = $!"