update script

src/broadcast_stage.rs

@@ -426,3 +426,4 @@ mod tests {
         assert_eq!(highest_index, 2 * leader_rotation_interval - 1);
     }
 }
+

src/loopfiles.py

@@ -13,21 +13,38 @@ def loopfile():
                     file_object.seek(0)
                     # If file is not empty then append '\n'
                     data = file_object.read(100)
-                    if len(data) > 0 :
-                        file_object.write("\n")
+
+                    for last_line in file_object:
+                        pass 
+                    print(filename)
+                    print(last_line)
+
+                    # If the file is updated in the last cycle
+                    if last_line == "999":
+                        remove_999()
+                    else:
+                        append_999()
+                    #if len(data) > 0 :
+                     #   file_object.write("\n")
                     # Append text at the end of file
-                    #file_object.write("hello hi")
+                    #file_object.write("9371")
 
         else:
             continue
 
+def remove_999():
+    print("Remove last line with 999")
+
+def append_999():
+    print("Append last line with 999")
+
 def commit():
     os.system('git commit -a -m "merge and update" > /dev/null 2>&1')
 
 def set_sys_time(year, month, day):
     os.system('date -s %04d%02d%02d' % (year, month, day))
 
 if __name__ == '__main__':
-    set_sys_time(2017, 1, 1)
+    # set_sys_time(2017, 1, 1)
     loopfile()
-    commit()
+    # commit()

src/update.py

@@ -2,7 +2,6 @@
 import os
 
 
-
 def modify():
     file = open('zero.md', 'r')
     flag = int(file.readline()) == 0
@@ -22,18 +21,49 @@ def loopfile():
             print(os.path.join(directory, filename))
 
             with open(filename, "a+") as file_object:
-                    # Move read cursor to the start of file.
-                    file_object.seek(0)
-                    # If file is not empty then append '\n'
-                    data = file_object.read(100)
-                    if len(data) > 0 :
-                        file_object.write("\n")
-                    # Append text at the end of file
-                    #file_object.write("hello hi")
-
+                # Move read cursor to the start of file.
+                file_object.seek(0)
+                # If file is not empty then append '\n'
+                data = file_object.read(100)
+
+                for last_line in file_object:
+                    pass
+                print(filename)
+                print(last_line)
+                file_object.close()
+                # If the file is updated in the last cycle
+                if last_line == " 01":
+                    remove_999(filename)
+                # else:
+                    # append_999(filename)
+
         else:
             continue
 
+
+def remove_999(filename):
+    print("Remove last line with 999")
+    fd = open(filename, "r")
+    d = fd.read()
+    fd.close()
+    m = d.split("\n")
+    s = "\n".join(m[:-1])
+    fd = open(filename, "w+")
+    for i in range(len(s)):
+        fd.write(s[i])
+    fd.close()
+
+
+def append_999(filename):
+    print("Append last line with 999")
+    with open(filename, "a+") as fil:
+        fil.seek(0)
+        data = fil.read(100)
+        if len(data) > 0:
+            fil.write("\n//999")
+    fil.close()
+
+
 def commit():
     os.system('git commit -a -m "merge and update" > /dev/null 2>&1')
 
@@ -55,4 +85,5 @@ def daily_commit(start_date, end_date):
 
 
 if __name__ == '__main__':
-    daily_commit(datetime.date(2017, 8, 31), datetime.date(2017, 12, 28))
+    # daily_commit(datetime.date(2017, 8, 31), datetime.date(2017, 12, 28))
+    loopfile()

test_ground/dummyfile1.rs

@@ -0,0 +1,189 @@
+use influx_db_client as influxdb;
+use metrics;
+use std::env;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use timing;
+
+const DEFAULT_METRICS_RATE: usize = 100;
+
+pub struct Counter {
+    pub name: &'static str,
+
+    pub counts: AtomicUsize,
+    pub times: AtomicUsize,
+
+    pub lastlog: AtomicUsize,
+    pub lograte: AtomicUsize,
+}
+
+macro_rules! create_counter {
+    ($name:expr, $lograte:expr) => {
+        Counter {
+            name: $name,
+            counts: AtomicUsize::new(0),
+            times: AtomicUsize::new(0),
+            lastlog: AtomicUsize::new(0),
+            lograte: AtomicUsize::new($lograte),
+        }
+    };
+}
+
+macro_rules! inc_counter {
+    ($name:expr, $count:expr) => {
+        unsafe { $name.inc($count) };
+    };
+}
+
+macro_rules! inc_new_counter_info {
+    ($name:expr, $count:expr) => {{
+        inc_new_counter!($name, $count, Level::Info, 0);
+    }};
+    ($name:expr, $count:expr, $lograte:expr) => {{
+        inc_new_counter!($name, $count, Level::Info, $lograte);
+    }};
+}
+
+macro_rules! inc_new_counter {
+    ($name:expr, $count:expr, $level:expr, $lograte:expr) => {{
+        if log_enabled!($level) {
+            static mut INC_NEW_COUNTER: Counter = create_counter!($name, $lograte);
+            inc_counter!(INC_NEW_COUNTER, $count);
+        }
+    }};
+}
+
+impl Counter {
+    fn default_log_rate() -> usize {
+        let v = env::var("XPZ_DEFAULT_METRICS_RATE")
+            .map(|x| x.parse().unwrap_or(DEFAULT_METRICS_RATE))
+            .unwrap_or(DEFAULT_METRICS_RATE);
+        if v == 0 {
+            DEFAULT_METRICS_RATE
+        } else {
+            v
+        }
+    }
+    pub fn inc(&mut self, events: usize) {
+        let counts = self.counts.fetch_add(events, Ordering::Relaxed);
+        let times = self.times.fetch_add(1, Ordering::Relaxed);
+        let mut lograte = self.lograte.load(Ordering::Relaxed);
+        if lograte == 0 {
+            lograte = Counter::default_log_rate();
+            self.lograte.store(lograte, Ordering::Relaxed);
+        }
+        if times % lograte == 0 && times > 0 {
+            let lastlog = self.lastlog.load(Ordering::Relaxed);
+            info!(
+                "COUNTER:{{\"name\": \"{}\", \"counts\": {}, \"samples\": {},  \"now\": {}, \"events\": {}}}",
+                self.name,
+                counts + events,
+                times,
+                timing::timestamp(),
+                events,
+            );
+            metrics::submit(
+                influxdb::Point::new(&format!("counter-{}", self.name))
+                    .add_field(
+                        "count",
+                        influxdb::Value::Integer(counts as i64 - lastlog as i64),
+                    ).to_owned(),
+            );
+            self.lastlog
+                .compare_and_swap(lastlog, counts, Ordering::Relaxed);
+        }
+    }
+}
+#[cfg(test)]
+mod tests {
+    use counter::{Counter, DEFAULT_METRICS_RATE};
+    use log::Level;
+    use std::env;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::{Once, RwLock, ONCE_INIT};
+
+    fn get_env_lock() -> &'static RwLock<()> {
+        static mut ENV_LOCK: Option<RwLock<()>> = None;
+        static INIT_HOOK: Once = ONCE_INIT;
+
+        unsafe {
+            INIT_HOOK.call_once(|| {
+                ENV_LOCK = Some(RwLock::new(()));
+            });
+            &ENV_LOCK.as_ref().unwrap()
+        }
+    }
+
+    #[test]
+    fn test_counter() {
+        let _readlock = get_env_lock().read();
+        static mut COUNTER: Counter = create_counter!("test", 100);
+        let count = 1;
+        inc_counter!(COUNTER, count);
+        unsafe {
+            assert_eq!(COUNTER.counts.load(Ordering::Relaxed), 1);
+            assert_eq!(COUNTER.times.load(Ordering::Relaxed), 1);
+            assert_eq!(COUNTER.lograte.load(Ordering::Relaxed), 100);
+            assert_eq!(COUNTER.lastlog.load(Ordering::Relaxed), 0);
+            assert_eq!(COUNTER.name, "test");
+        }
+        for _ in 0..199 {
+            inc_counter!(COUNTER, 2);
+        }
+        unsafe {
+            assert_eq!(COUNTER.lastlog.load(Ordering::Relaxed), 199);
+        }
+        inc_counter!(COUNTER, 2);
+        unsafe {
+            assert_eq!(COUNTER.lastlog.load(Ordering::Relaxed), 399);
+        }
+    }
+    #[test]
+    fn test_inc_new_counter() {
+        let _readlock = get_env_lock().read();
+        inc_new_counter_info!("counter-1", 1);
+        inc_new_counter_info!("counter-2", 1, 2);
+    }
+    #[test]
+    fn test_lograte() {
+        let _readlock = get_env_lock().read();
+        assert_eq!(
+            Counter::default_log_rate(),
+            DEFAULT_METRICS_RATE,
+            "default_log_rate() is {}, expected {}, XPZ_DEFAULT_METRICS_RATE environment variable set?",
+            Counter::default_log_rate(),
+            DEFAULT_METRICS_RATE,
+        );
+        static mut COUNTER: Counter = create_counter!("test_lograte", 0);
+        inc_counter!(COUNTER, 2);
+        unsafe {
+            assert_eq!(
+                COUNTER.lograte.load(Ordering::Relaxed),
+                DEFAULT_METRICS_RATE
+            );
+        }
+    }
+
+    #[test]
+    fn test_lograte_env() {
+        assert_ne!(DEFAULT_METRICS_RATE, 0);
+        let _writelock = get_env_lock().write();
+        static mut COUNTER: Counter = create_counter!("test_lograte_env", 0);
+        env::set_var("XPZ_DEFAULT_METRICS_RATE", "50");
+        inc_counter!(COUNTER, 2);
+        unsafe {
+            assert_eq!(COUNTER.lograte.load(Ordering::Relaxed), 50);
+        }
+
+        static mut COUNTER2: Counter = create_counter!("test_lograte_env", 0);
+        env::set_var("XPZ_DEFAULT_METRICS_RATE", "0");
+        inc_counter!(COUNTER2, 2);
+        unsafe {
+            assert_eq!(
+                COUNTER2.lograte.load(Ordering::Relaxed),
+                DEFAULT_METRICS_RATE
+            );
+        }
+    }
+}
+
+ 01