add symbolic link support for snapshot

pom.xml

@@ -4,7 +4,7 @@
 
  <groupId>com.github.wenweihu86.raft</groupId>
  <artifactId>raft-java-all</artifactId>
- <version>1.3.0</version>
+ <version>1.4.0</version>
  <packaging>pom</packaging>
 
  <name>raft-java-all</name>

raft-java-admin/pom.xml

@@ -5,7 +5,7 @@
  <modelVersion>4.0.0</modelVersion>
  <groupId>com.github.wenweihu86.raft</groupId>
  <artifactId>raft-java-admin</artifactId>
- <version>1.3.0</version>
+ <version>1.4.0</version>
  <packaging>jar</packaging>
 
  <name>raft-java-admin</name>

raft-java-admin/start_admin.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
-java -cp dependency/*:raft-java-admin-1.3.0.jar com.github.wenweihu86.raft.admin.AdminMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053" conf get
+java -cp dependency/*:raft-java-admin-1.4.0.jar com.github.wenweihu86.raft.admin.AdminMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053" conf get
 
-java -cp dependency/*:raft-java-admin-1.3.0.jar com.github.wenweihu86.raft.admin.AdminMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053" conf add "127.0.0.1:8054:4,127.0.0.1:8055:5"
+java -cp dependency/*:raft-java-admin-1.4.0.jar com.github.wenweihu86.raft.admin.AdminMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053" conf add "127.0.0.1:8054:4,127.0.0.1:8055:5"
 
-java -cp dependency/*:raft-java-admin-1.3.0.jar com.github.wenweihu86.raft.admin.AdminMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053" conf del "127.0.0.1:8054:4,127.0.0.1:8055:5"
+java -cp dependency/*:raft-java-admin-1.4.0.jar com.github.wenweihu86.raft.admin.AdminMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053" conf del "127.0.0.1:8054:4,127.0.0.1:8055:5"

raft-java-core/pom.xml

@@ -4,7 +4,7 @@
 
  <groupId>com.github.wenweihu86.raft</groupId>
  <artifactId>raft-java-core</artifactId>
- <version>1.3.0</version>
+ <version>1.4.0</version>
  <packaging>jar</packaging>
 
  <name>raft-java-core</name>

raft-java-core/src/main/java/com/github/wenweihu86/raft/RaftNode.java

@@ -100,7 +100,9 @@ public RaftNode(List<RaftMessage.Server> servers, RaftMessage.Server localServer
  }
  }
  lastAppliedIndex = commitIndex;
+ }
 
+ public void init() {
  for (RaftMessage.Server server : configuration.getServersList()) {
  if (!peerMap.containsKey(server.getServerId())
  && server.getServerId() != localServer.getServerId()) {
@@ -124,9 +126,6 @@ public void run() {
  takeSnapshot();
  }
  }, RaftOptions.snapshotPeriodSeconds, RaftOptions.snapshotPeriodSeconds, TimeUnit.SECONDS);
- }
-
- public void init() {
  // start election
  resetElectionTimer();
  }
@@ -662,7 +661,7 @@ public void stepDown(long newTerm) {
  resetElectionTimer();
  }
 
- private void takeSnapshot() {
+ public void takeSnapshot() {
  if (!snapshot.getIsInSnapshot().compareAndSet(false, true)) {
  LOG.info("already in snapshot, ignore takeSnapshot");
  return;

raft-java-core/src/main/java/com/github/wenweihu86/raft/storage/SegmentedLog.java

@@ -170,6 +170,7 @@ public void truncatePrefix(long newFirstIndex) {
  try {
  RaftFileUtils.closeFile(segment.getRandomAccessFile());
  FileUtils.forceDelete(oldFile);
+ totalSize -= segment.getFileSize();
  startLogIndexSegmentMap.remove(segment.getStartIndex());
  } catch (Exception ex2) {
  LOG.warn("delete file exception:", ex2);

raft-java-core/src/main/java/com/github/wenweihu86/raft/storage/Snapshot.java

@@ -11,6 +11,8 @@
 import java.io.IOException;
 import java.io.RandomAccessFile;
 
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
 import java.util.List;
 import java.util.TreeMap;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -60,9 +62,18 @@ public void reload() {
  }
  }
 
+ // 如果是软链接，需要打开实际文件句柄
  public TreeMap<String, SnapshotDataFile> readSnapshotDataFiles() {
  TreeMap<String, SnapshotDataFile> snapshotDataFileMap = new TreeMap<>();
  String snapshotDataDir = snapshotDir + File.separator + "data";
+ try {
+ Path snapshotDataPath = FileSystems.getDefault().getPath(snapshotDataDir);
+ snapshotDataPath = snapshotDataPath.toRealPath();
+ snapshotDataDir = snapshotDataPath.toString();
+ } catch (IOException ex) {
+ LOG.warn("readSnapshotDataFiles exception:", ex);
+ throw new RuntimeException(ex);
+ }
  List<String> fileNames = RaftFileUtils.getSortedFilesInDirectory(snapshotDataDir);
  for (String fileName : fileNames) {
  RandomAccessFile randomAccessFile = RaftFileUtils.openFile(snapshotDataDir, fileName, "r");

raft-java-core/src/test/java/com/github/wenweihu86/raft/storage/SnapshotTest.java

@@ -0,0 +1,41 @@
+package com.github.wenweihu86.raft.storage;
+
+import com.github.wenweihu86.raft.RaftOptions;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.TreeMap;
+
+/**
+ * Created by wenweihu86 on 2017/7/2.
+ */
+public class SnapshotTest {
+
+ @Test
+ public void testReadSnapshotDataFiles() throws IOException {
+ RaftOptions.dataDir = "./data";
+ File file = new File("./data/message");
+ file.mkdirs();
+ File file1 = new File("./data/message/queue1.txt");
+ file1.createNewFile();
+ File file2 = new File("./data/message/queue2.txt");
+ file2.createNewFile();
+
+ File snapshotFile = new File("./data/snapshot");
+ snapshotFile.mkdirs();
+ Path link = FileSystems.getDefault().getPath("./data/snapshot/data");
+ Path target = FileSystems.getDefault().getPath("./data/message").toRealPath();
+ Files.createSymbolicLink(link, target);
+
+ Snapshot snapshot = new Snapshot();
+ TreeMap<String, Snapshot.SnapshotDataFile> snapshotFileMap = snapshot.readSnapshotDataFiles();
+ System.out.println(snapshotFileMap.keySet());
+ Assert.assertTrue(snapshotFileMap.size() == 2);
+ Assert.assertTrue(snapshotFileMap.firstKey().equals("queue1.txt"));
+ }
+}

raft-java-example/pom.xml

@@ -4,7 +4,7 @@
 
  <groupId>com.github.wenweihu86.raft</groupId>
  <artifactId>raft-java-example</artifactId>
- <version>1.3.0</version>
+ <version>1.4.0</version>
  <packaging>jar</packaging>
 
  <name>raft-java-example</name>

raft-java-example/start_client.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 
-java -cp dependency/*:raft-java-example-1.3.0.jar com.github.wenweihu86.raft.example.client.ClientMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053" hello raft
+java -cp dependency/*:raft-java-example-1.4.0.jar com.github.wenweihu86.raft.example.client.ClientMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053" hello raft
 
-java -cp dependency/*:raft-java-example-1.3.0.jar com.github.wenweihu86.raft.example.client.ConcurrentClientMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053"
+java -cp dependency/*:raft-java-example-1.4.0.jar com.github.wenweihu86.raft.example.client.ConcurrentClientMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053"

raft-java-example/start_server.sh

@@ -1,11 +1,11 @@
 #!/usr/bin/env bash
 
-java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example1/data -cp dependency/*:raft-java-example-1.3.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8051:1"
+java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example1/data -cp dependency/*:raft-java-example-1.4.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8051:1"
 
-java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example2/data -cp dependency/*:raft-java-example-1.3.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8052:2"
+java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example2/data -cp dependency/*:raft-java-example-1.4.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8052:2"
 
-java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example3/data -cp dependency/*:raft-java-example-1.3.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8053:3"
+java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example3/data -cp dependency/*:raft-java-example-1.4.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8053:3"
 
-java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example4/data -cp dependency/*:raft-java-example-1.3.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8054:4"
+java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example4/data -cp dependency/*:raft-java-example-1.4.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8054:4"
 
-java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example5/data -cp dependency/*:raft-java-example-1.3.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8055:5"
+java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example5/data -cp dependency/*:raft-java-example-1.4.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8055:5"