v1.0.0 timeFormat added

.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+ // See http:https://go.microsoft.com/fwlink/?LinkId=827846
+ // for the documentation about the extensions.json format
+ "recommendations": [
+ "dbaeumer.vscode-eslint"
+ ]
+}

.vscode/launch.json

@@ -0,0 +1,36 @@
+// A launch configuration that compiles the extension and then opens it inside a new window
+// Use IntelliSense to learn about possible attributes.
+// Hover to view descriptions of existing attributes.
+// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+{
+ "version": "0.2.0",
+ "configurations": [
+ {
+ "name": "Run Extension",
+ "type": "extensionHost",
+ "request": "launch",
+ "runtimeExecutable": "${execPath}",
+ "args": [
+ "--extensionDevelopmentPath=${workspaceFolder}"
+ ],
+ "outFiles": [
+ "${workspaceFolder}/out/**/*.js"
+ ],
+ "preLaunchTask": "${defaultBuildTask}"
+ },
+ {
+ "name": "Extension Tests",
+ "type": "extensionHost",
+ "request": "launch",
+ "runtimeExecutable": "${execPath}",
+ "args": [
+ "--extensionDevelopmentPath=${workspaceFolder}",
+ "--extensionTestsPath=${workspaceFolder}/out/test/suite/index"
+ ],
+ "outFiles": [
+ "${workspaceFolder}/out/test/**/*.js"
+ ],
+ "preLaunchTask": "${defaultBuildTask}"
+ }
+ ]
+}

.vscode/settings.json

@@ -0,0 +1,11 @@
+// Place your settings in this file to overwrite default and user settings.
+{
+ "files.exclude": {
+ "out": false // set this to true to hide the "out" folder with the compiled JS files
+ },
+ "search.exclude": {
+ "out": true // set this to false to include "out" folder in search results
+ },
+ // Turn off tsc task auto detection since we have the necessary tasks as npm scripts
+ "typescript.tsc.autoDetect": "off"
+}

.vscode/tasks.json

@@ -0,0 +1,20 @@
+// See https://go.microsoft.com/fwlink/?LinkId=733558
+// for the documentation about the tasks.json format
+{
+ "version": "2.0.0",
+ "tasks": [
+ {
+ "type": "npm",
+ "script": "watch",
+ "problemMatcher": "$tsc-watch",
+ "isBackground": true,
+ "presentation": {
+ "reveal": "never"
+ },
+ "group": {
+ "kind": "build",
+ "isDefault": true
+ }
+ }
+ ]
+}

CHANGELOG.md

@@ -3,6 +3,11 @@
 All notable changes to the "smart-log" extension will be documented in this file.
 
 <!-- Check [Keep a Changelog](http:https://keepachangelog.com/) for recommendations on how to structure this file. -->
+## [1.0.0]
+
+- improved time regex / parsing
+- added example for "common log format" (http logs)
+- post time updates only on valid times
 
 ## [0.9.1]
 - Added telemetry using vscode-extension-telemetry with events: 'activate' and 'open file' (measurements number of fileConfigs).

README.md

@@ -4,6 +4,8 @@
 
 This Visual Studio Code(tm) extension adds **analysis** and **visualization** features for log files and should ease debugging especially of complex systems where multiple log files need to be considered at the same time.
 
+![Smart-Log in action](./images/smart-log_1.gif)
+
 **Note:** It works well with [![Version](https://vsmarketplacebadge.apphb.com/version/mbehr1.vsc-lfs.svg)](https://marketplace.visualstudio.com/items?itemName=mbehr1.vsc-lfs) to handle large log files (few hundred MBs).
 
 **Note:** The **time-sync** feature works well with [![Version](https://vsmarketplacebadge.apphb.com/version/mbehr1.dlt-logs.svg)](https://marketplace.visualstudio.com/items?itemName=mbehr1.dlt-logs) for DLT (diagnostic log and trace) files.
@@ -22,14 +24,6 @@ This Visual Studio Code(tm) extension adds **analysis** and **visualization** fe
  - Allows to "synchronize" this time with other visible documents from all plugins that support "onDidChangeSelectedTime" events. 
 - Allows configuration of **multiple file types** (see settings smart-log.fileConfigs).
 
-<!-- 
-For example if there is an image subfolder under your extension project workspace:
-
-\!\[feature X\]\(images/feature-x.png\)
-
-> Tip: Many popular extensions utilize animations. This is an excellent way to show off your extension! We recommend short, focused animations that are easy to follow.
--->
-
 ## Requirements
 
 To open large log files please consider installing "vsc-lfs" extension.
@@ -38,6 +32,7 @@ To open large log files please consider installing "vsc-lfs" extension.
 This extension contributes the following settings:
 
 * `smart-log.timeRegex`: Regular expression (regex) to identify the time of a log line.
+* `smart-log.timeFormat`: Optional time format specifier (details see config example).
 * `smart-log.decorations`: Definition of the decoration types supported for events.
 * `smart-log.fileConfigs`: Definition of the configuration per file type.
 
@@ -56,10 +51,11 @@ Open an [issue](https://github.com/mbehr1/smart-log/issues) or create a pull req
 
 ## Planned features
 
+* add filtering based on events (remove all but the events)
 * automatic timezone detection
-* improved time regex / parsing
 * time sync auto-synchronization
 * change "smart-log" file type automatically to sub-types
+* change file type back to e.g. Log or Plain text on if no config is found.
 * add event icon support in tree view
 * add event support to modify parent title
 * add decoration for time-sync
@@ -68,6 +64,14 @@ Open an [issue](https://github.com/mbehr1/smart-log/issues) or create a pull req
 
 See [Changelog](./CHANGELOG.md)
 
+## Third-party Content
+
+This project leverages the following third party content.
+
+d3-time-format (2.2.3)
+ - License: BSD-Clause-3 https://github.com/d3/d3-time-format/blob/master/LICENSE
+ - Source: https://github.com/d3/d3-time-format
+
 <!-- 
 **Note:** You can author your README using Visual Studio Code. Here are some useful editor keyboard shortcuts:
 

package.json

@@ -2,7 +2,7 @@
  "name": "smart-log",
  "displayName": "Smart-Log",
  "description": "Provides analysis and visualization features for text based log files.",
- "version": "0.9.1",
+ "version": "1.0.0",
  "license": "CC BY-NC-SA 4.0",
  "publisher": "mbehr1",
  "author": {
@@ -36,6 +36,10 @@
  "description": "Regular expression used to identify the time for a line of the logs. This can be overwritten by timeRegex setting inside each fileConfig. Needs to have at least 6 captures (year, month, date, hour, min, sec) can have a 7th (ms) or only 1 capture (time string that Date() accepts).",
  "default": "^([0-2][0-9]|[0-2][0-9][0-9][0-9])\\-([0-1][0-9])\\-([0-3][0-9]) ([0-2][0-9])\\:([0-5][0-9])\\:([0-5][0-9]),([0-9][0-9][0-9])"
  },
+ "smart-log.timeFormat": {
+ "type": "string",
+ "description": "Optional time format specifier (details see d3-time-format). If this is provided the timeRegex has to capture one expression that will be parsed according to this specifier. Example '%d/%b/%Y:%H:%M:%S %Z'. If not provided the default Date() constructor will be used."
+ },
  "smart-log.decorations": {
  "type": "array",
  "items": {
@@ -118,6 +122,10 @@
  "type": "string",
  "description": "Regexp used to identify the time for a line of the logs. If provided it overwrittes the smart-log.timeRegex setting."
  },
+ "timeFormat": {
+ "type": "string",
+ "description": "Optional time format specifier (details see d3-time-format). If this is provided the timeRegex has to capture one expression that will be parsed according to this specifier. Example '%d/%b/%Y:%H:%M:%S %Z'. If not provided the default Date() constructor will be used."
+ },
  "events": {
  "type": "array",
  "items": {
@@ -147,6 +155,26 @@
  }
  },
  "default": [
+ {
+ "name": "common log format",
+ "identifyRegex": "HTTP/",
+ "timeRegex": "^\\S+ \\S+ \\S+ \\[([^\\]]+)",
+ "timeFormat": "%d/%b/%Y:%H:%M:%S %Z",
+ "events": [
+ {
+ "level": 1,
+ "label": "error {2} {1}",
+ "decorationId": "error",
+ "regex": "^\\S+ \\S+ \\S+ \\[[^\\]]+\\] \"([A-Z]+ [^ \"]+? HTTP/[0-9.]+)\" (5[0-9]{2}) "
+ },
+ {
+ "level": 2,
+ "label": "{2} {1}",
+ "decorationId": "warning",
+ "regex": "^\\S+ \\S+ \\S+ \\[[^\\]]+\\] \"([A-Z]+ [^ \"]+? HTTP/[0-9.]+)\" (4[0-9]{2}) "
+ }
+ ]
+ },
  {
  "name": "syslog",
  "identifyRegex": "\\s(.*)\\[(\\d*)\\]:",
@@ -235,6 +263,8 @@
  "vscode-test": "^1.3.0"
  },
  "dependencies": {
+ "@types/d3-time-format": "^2.1.1",
+ "d3-time-format": "^2.2.3",
  "vscode-extension-telemetry": "^0.1.2"
  }
 }

src/extension.ts

@@ -5,6 +5,7 @@
 import * as vscode from 'vscode';
 import * as path from 'path';
 import TelemetryReporter from 'vscode-extension-telemetry';
+import * as d3 from 'd3-time-format';
 
 let reporter: TelemetryReporter;
 
@@ -81,6 +82,7 @@ interface DataPerDocument {
  identifiedFileConfig?: any
  cachedTimes?: Array<Date>; // per line one date/time
  timeRegex?: RegExp; // from file config or default
+ timeFormat?: string;
  timeAdjustMs?: number; // adjust in ms
 };
 export default class SmartLogs implements vscode.TreeDataProvider<EventNode>, vscode.Disposable {
@@ -99,6 +101,8 @@ export default class SmartLogs implements vscode.TreeDataProvider<EventNode>, vs
  private _defaultTimeRegex = new RegExp('^([0-2][0-9]|[0-2][0-9][0-9][0-9])\-([0-1][0-9])\-([0-3][0-9]) ([0-2][0-9])\:([0-5][0-9])\:([0-5][0-9]),([0-9][0-9][0-9])');
  private _timeRegex = vscode.workspace.getConfiguration().get<string>("smart-log.timeRegex") ?
  new RegExp(<string>(vscode.workspace.getConfiguration().get<string>("smart-log.timeRegex"))) : this._defaultTimeRegex;
+ private _timeFormat = vscode.workspace.getConfiguration().get<string>("smart-log.timeFormat");
+
 
  private _onDidChangeTreeData: vscode.EventEmitter<EventNode | null> = new vscode.EventEmitter<EventNode | null>();
  readonly onDidChangeTreeData: vscode.Event<EventNode | null> = this._onDidChangeTreeData.event;
@@ -118,10 +122,12 @@ export default class SmartLogs implements vscode.TreeDataProvider<EventNode>, vs
  this._fileConfigs = vscode.workspace.getConfiguration().get<Array<object>>("smart-log.fileConfigs");
  this._timeRegex = vscode.workspace.getConfiguration().get<string>("smart-log.timeRegex") ?
  new RegExp(<string>(vscode.workspace.getConfiguration().get<string>("smart-log.timeRegex"))) : this._defaultTimeRegex;
+ this._timeFormat = vscode.workspace.getConfiguration().get<string>("smart-log.timeFormat");
  console.log(` #fileConfigs=${this._fileConfigs?.length}`);
  this._documents.forEach((data) => {
  data.identifiedFileConfig = undefined; // reset here to let config changes apply
  data.timeRegex = undefined;
+ data.timeFormat = undefined;
  data.cachedTimes = undefined;
  this.updateData(data);
  });
@@ -191,8 +197,10 @@ export default class SmartLogs implements vscode.TreeDataProvider<EventNode>, vs
  // determine time:
  const time = this.provideTimeByData(data, line);
  // post time update...
- console.log(` smart-log posting time update ${time.toLocaleTimeString()}.${String(time.valueOf() % 1000).padStart(3, "0")}`);
- this._onDidChangeSelectedTime.fire({ time: time, uri: data.doc.uri });
+ if (time.valueOf() > 0) {
+ console.log(` smart-log posting time update ${time.toLocaleTimeString()}.${String(time.valueOf() % 1000).padStart(3, "0")}`);
+ this._onDidChangeSelectedTime.fire({ time: time, uri: data.doc.uri });
+ }
  }
  }
  }));
@@ -383,6 +391,12 @@ export default class SmartLogs implements vscode.TreeDataProvider<EventNode>, vs
  console.log("smart-log.provideTimeByData has no timeRegex!");
  return new Date(0);
  }
+
+ let d3TimeParser = null;
+ if (data.timeFormat) {
+ d3TimeParser = d3.timeParse(data.timeFormat);
+ }
+
  // we reset the times here in any case:
  data.cachedTimes = new Array<Date>();
 
@@ -398,9 +412,20 @@ export default class SmartLogs implements vscode.TreeDataProvider<EventNode>, vs
  const ms: number = regRes[7] ? +regRes[7] : 0;
  let date = new Date(year, +regRes[2] - 1, +regRes[3], +regRes[4], +regRes[5], +regRes[6], ms);
  data.cachedTimes.push(date);
- } else if (regRes.length === 2) {
- let date = new Date(regRes[1]);
- data.cachedTimes.push(date);
+ } else if (regRes.length === 2) { // one complete date string
+ let date: Date | null = null;
+ if (d3TimeParser) {
+ try {
+ const parsedTime = d3TimeParser(regRes[1]);
+ // console.log(`got parsedTime ${parsedTime}`);
+ date = <Date>parsedTime;
+ } catch (error) {
+ console.log(`got error ${error}`);
+ }
+ } else {
+ date = new Date(regRes[1]);
+ }
+ data.cachedTimes.push(date ? date : new Date(0));
  }
  } else {
  // use the one from prev. line
@@ -484,12 +509,17 @@ export default class SmartLogs implements vscode.TreeDataProvider<EventNode>, vs
 
  if (identifiedFileConfig) {
 
- if (identifiedFileConfig.timeRegex) {
+ if ("timeRegex" in identifiedFileConfig) {
  data.timeRegex = new RegExp(<string>identifiedFileConfig.timeRegex);
  } else {
  console.log(" using default timeRegex");
  data.timeRegex = this._timeRegex;
  }
+ if ("timeFormat" in identifiedFileConfig) {
+ data.timeFormat = identifiedFileConfig.timeFormat;
+ } else {
+ data.timeFormat = this._timeFormat;
+ }
 
  const events: any | undefined = identifiedFileConfig.events;
  // create the RegExps here to have them compiled and not created line by line