store att for aakd

DefangChen · Aug 5, 2020 · 85140ad · 85140ad
1 parent ab84437
commit 85140ad
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 17 deletions.
diff --git a/models/util.py b/models/util.py
@@ -54,7 +54,7 @@ def forward(self, x):
 
 class SelfA(nn.Module):
  """Cross layer Self Attention"""
- def __init__(self, s_len, t_len, input_channel, s_n, s_t, factor=2): 
+ def __init__(self, s_len, t_len, input_channel, s_n, s_t, factor=4): 
  super(SelfA, self).__init__()
 
  self.avgpool = nn.AdaptiveAvgPool2d((1,1))

diff --git a/results/records.md b/results/records.md
@@ -90,4 +90,10 @@ python train_student.py --path-t ./save/models/ResNet34_vanilla/resnet34_transfo
 这个一开始爆显存了，所以 DALI 都用了 cpu 来减少显存用量。
 ## aakd
 7.30 早上十点多开始，开始的 GPU 用量：
-![vgg_shuffle_aakd_GPU](vgg_shuffle_aakd_GPU.png)
+![vgg_shuffle_aakd_GPU](vgg_shuffle_aakd_GPU.png)
+
+# 懒得写记录了 直接画表格
+VGG13 -> ShuffleV2
+|aakd(b=100)|aakd(b=400)|hint(re-run required)|irg(b=0.05)|sp|vid|
+|-|-|-|-|-|-|
+|63.228|62.144|63.134|63.82x|
diff --git a/train_student.py b/train_student.py
@@ -430,6 +430,8 @@ def main_worker(gpu, ngpus_per_node, opt):
  'model': model_s.state_dict(),
  'best_acc': best_acc,
  }
+ if opt.distill == 'aakd':
+ state['attention'] = trainable_list[-1].state_dict()
  save_file = os.path.join(opt.save_folder, '{}_best.pth'.format(opt.model_s))
 
  test_merics = {'test_loss': test_loss,

diff --git a/vgg13_shufflev2.sh b/vgg13_shufflev2.sh
@@ -5,25 +5,25 @@ EPOCH=90
 LEARNING_RATE=0.1
 DALI=cpu
 
-# kd
+# # kd
 # python train_student.py --path-t ./save/models/vgg13_imagenet_vanilla/vgg13_transformed.pth \
 # --batch_size $BATCH_SIZE --epochs $EPOCH --dataset imagenet --gpu_id $GPU --dist-url tcp:https://127.0.0.1:23333 \
 # --print-freq 100 --num_workers $WORKER --model_s ShuffleV2_Imagenet --trial release \
 # --multiprocessing-distributed --learning_rate $LEARNING_RATE --lr_decay_epochs 30,60 --weight_decay 1e-4 --dali $DALI \
 # --distill kd -r 1 -a 1 -b 0
-# fitnet
-python train_student.py --path-t ./save/models/vgg13_imagenet_vanilla/vgg13_transformed.pth \
---batch_size $BATCH_SIZE --epochs $EPOCH --dataset imagenet --gpu_id $GPU --dist-url tcp:https://127.0.0.1:23333 \
---print-freq 100 --num_workers $WORKER --model_s ShuffleV2_Imagenet --trial release \
---multiprocessing-distributed --learning_rate $LEARNING_RATE --lr_decay_epochs 30,60 --weight_decay 1e-4 --dali $DALI \
---distill hint -r 1 -a 1 -b 100 --hint_layer 1
-# sp
-python train_student.py --path-t ./save/models/vgg13_imagenet_vanilla/vgg13_transformed.pth \
---batch_size $BATCH_SIZE --epochs $EPOCH --dataset imagenet --gpu_id $GPU --dist-url tcp:https://127.0.0.1:23333 \
---print-freq 100 --num_workers $WORKER --model_s ShuffleV2_Imagenet --trial release \
---multiprocessing-distributed --learning_rate $LEARNING_RATE --lr_decay_epochs 30,60 --weight_decay 1e-4 --dali $DALI \
---distill similarity -r 1 -a 1 -b 3000
-# vid
+# # fitnet
+# python train_student.py --path-t ./save/models/vgg13_imagenet_vanilla/vgg13_transformed.pth \
+# --batch_size $BATCH_SIZE --epochs $EPOCH --dataset imagenet --gpu_id $GPU --dist-url tcp:https://127.0.0.1:23333 \
+# --print-freq 100 --num_workers $WORKER --model_s ShuffleV2_Imagenet --trial release \
+# --multiprocessing-distributed --learning_rate $LEARNING_RATE --lr_decay_epochs 30,60 --weight_decay 1e-4 --dali $DALI \
+# --distill hint -r 1 -a 1 -b 100 --hint_layer 1
+# # sp
+# python train_student.py --path-t ./save/models/vgg13_imagenet_vanilla/vgg13_transformed.pth \
+# --batch_size $BATCH_SIZE --epochs $EPOCH --dataset imagenet --gpu_id $GPU --dist-url tcp:https://127.0.0.1:23333 \
+# --print-freq 100 --num_workers $WORKER --model_s ShuffleV2_Imagenet --trial release \
+# --multiprocessing-distributed --learning_rate $LEARNING_RATE --lr_decay_epochs 30,60 --weight_decay 1e-4 --dali $DALI \
+# --distill similarity -r 1 -a 1 -b 3000
+# # vid
 # python train_student.py --path-t ./save/models/vgg13_imagenet_vanilla/vgg13_transformed.pth \
 # --batch_size $BATCH_SIZE --epochs $EPOCH --dataset imagenet --gpu_id $GPU --dist-url tcp:https://127.0.0.1:23333 \
 # --print-freq 100 --num_workers $WORKER --model_s ShuffleV2_Imagenet --trial release \
@@ -35,7 +35,7 @@ python train_student.py --path-t ./save/models/vgg13_imagenet_vanilla/vgg13_tran
 --print-freq 100 --num_workers $WORKER --model_s ShuffleV2_Imagenet --trial release \
 --multiprocessing-distributed --learning_rate $LEARNING_RATE --lr_decay_epochs 30,60 --weight_decay 1e-4 --dali $DALI \
 --distill aakd -r 1 -a 1 -b 100
-# irg
+# # irg
 # python train_student.py --path-t ./save/models/vgg13_imagenet_vanilla/vgg13_transformed.pth \
 # --batch_size $BATCH_SIZE --epochs $EPOCH --dataset imagenet --gpu_id $GPU --dist-url tcp:https://127.0.0.1:23333 \
 # --print-freq 100 --num_workers $WORKER --model_s ShuffleV2_Imagenet --trial release \