donot skip non extension files (deepfence#86)

* donot skip non extension files
* make info to debug

core/match.go

@@ -17,16 +17,18 @@ type MatchFile struct {
 
 // IsSkippableDir Checks if the path is excluded
 func IsSkippableDir(excludedPaths []string, path, baseDir string) bool {
-
  for _, skippablePathIndicator := range excludedPaths {
  if strings.HasPrefix(path, skippablePathIndicator) || strings.HasPrefix(path, filepath.Join(baseDir, skippablePathIndicator)) {
+ log.Debugf("Path %s is skippable", path)
  return true
  }
  if strings.Contains(path, skippablePathIndicator) || strings.Contains(path, filepath.Join(baseDir, skippablePathIndicator)) {
+ log.Debugf("Path %s is skippable", path)
  return true
  }
  }
 
+ log.Debugf("Path %s is not skippable", path)
  return false
 }
 

go.mod

@@ -10,7 +10,11 @@ require (
  github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20230929125743-1713a043efe5
  github.com/deepfence/vessel v0.12.3
  github.com/gabriel-vasile/mimetype v1.4.3
+ github.com/gorilla/handlers v1.5.2
+ github.com/gorilla/mux v1.8.1
+ github.com/hillu/go-yara v1.3.1
  github.com/hillu/go-yara/v4 v4.3.2
+ github.com/jheise/yaramsg v0.0.0-20161011054113-41444562d276
  github.com/olekukonko/tablewriter v0.0.5
  github.com/sirupsen/logrus v1.9.3
  google.golang.org/grpc v1.63.2

go.sum

@@ -92,6 +92,10 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE=
+github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
@@ -100,8 +104,12 @@ github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxC
 github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
 github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
 github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
+github.com/hillu/go-yara v1.3.1 h1:DlIPDhRDXmbV2gFV9yhGpt10wzTprnOXI5bN0XIWs6k=
+github.com/hillu/go-yara v1.3.1/go.mod h1:KLxCsvD3F8cgVK866UDHi961qbzP+twKjhNdDsuz/2M=
 github.com/hillu/go-yara/v4 v4.3.2 h1:HGqUN3ORUduWZbb95RQjut4UzavGDbtt/C6SnGB3Amk=
 github.com/hillu/go-yara/v4 v4.3.2/go.mod h1:AHEs/FXVMQKVVlT6iG9d+q1BRr0gq0WoAWZQaZ0gS7s=
+github.com/jheise/yaramsg v0.0.0-20161011054113-41444562d276 h1:pq9AaNjlA2Oiljorl9+QxZR6GmzIYbZe9/EoB2L3gxk=
+github.com/jheise/yaramsg v0.0.0-20161011054113-41444562d276/go.mod h1:/+N5vT6ah10PM3rvPV2E4noyNb4CDTWm0PFnQOL/grA=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=

pkg/scan/process_image.go

@@ -213,6 +213,7 @@ func isExecutable(path string) bool {
 }
 
 func ScanFile(s *Scanner, f *os.File, iocs *[]output.IOCFound, layer string) error {
+ logrus.Debugf("Scanning file %s", f.Name())
  var (
  matches yr.MatchRules
  err error
@@ -230,130 +231,127 @@ func ScanFile(s *Scanner, f *os.File, iocs *[]output.IOCFound, layer string) err
  return nil
  }
 
- if filepath.Ext(f.Name()) != "" {
- variables := []ruleVariable{
- {"filename", filepath.ToSlash(filepath.Base(f.Name()))},
- {"filepath", filepath.ToSlash(f.Name())},
- {"extension", filepath.Ext(f.Name())},
- }
+ variables := []ruleVariable{
+ {"filename", filepath.ToSlash(filepath.Base(f.Name()))},
+ {"filepath", filepath.ToSlash(f.Name())},
+ {"extension", filepath.Ext(f.Name())},
+ }
 
- yrScanner := s.YaraScanner
- yrScanner.SetCallback(&matches)
- for _, v := range variables {
- if v.value != nil {
- if err = yrScanner.DefineVariable(v.name, v.value); err != nil {
- return filepath.SkipDir
- }
+ yrScanner := s.YaraScanner
+ yrScanner.SetCallback(&matches)
+ for _, v := range variables {
+ if v.value != nil {
+ if err = yrScanner.DefineVariable(v.name, v.value); err != nil {
+ return filepath.SkipDir
  }
  }
+ }
 
- fi, err := f.Stat()
+ fi, err := f.Stat()
+ if err != nil {
+ // report.AddStringf("yara: %s: Error accessing file information, error=%s",
+ // f.Name(), err.Error())
+ return err
+ }
+ fileName := f.Name()
+ hostMountPath := *s.HostMountPath
+ if hostMountPath != "" {
+ fileName = strings.TrimPrefix(fileName, hostMountPath)
+ }
+ if *s.MaximumFileSize > 0 && fi.Size() > *s.MaximumFileSize {
+ logrus.Debugf("\nyara: %v: Skipping large file, size=%v, max_size=%v", fileName, fi.Size(), *s.MaximumFileSize)
+ return nil
+ }
+ err = yrScanner.ScanFileDescriptor(f.Fd())
+ if err != nil {
+ fmt.Println("Scan File Descriptor error, trying alternative", err)
+ var buf []byte
+ if buf, err = io.ReadAll(f); err != nil {
+ logrus.Errorf("yara: %s: Error reading file, error=%s",
+ fileName, err.Error())
+ return filepath.SkipDir
+ }
+ err = yrScanner.ScanMem(buf)
  if err != nil {
- // report.AddStringf("yara: %s: Error accessing file information, error=%s",
- // f.Name(), err.Error())
- return err
+ fmt.Println("Scan File Mmory Error", err)
+ return filepath.SkipDir
  }
- fileName := f.Name()
- hostMountPath := *s.HostMountPath
- if hostMountPath != "" {
- fileName = strings.TrimPrefix(fileName, hostMountPath)
+
+ }
+ var iocsFound []output.IOCFound
+ totalMatchesStringData := make([]string, 0)
+ for _, m := range matches {
+ matchesStringData := make([]string, len(m.Strings))
+ for _, str := range m.Strings {
+ if !strings.Contains(strings.Join(matchesStringData, " "), string(str.Data)) {
+ matchesStringData = append(matchesStringData, string(str.Data))
+ totalMatchesStringData = append(totalMatchesStringData, string(str.Data))
+ }
  }
- if *s.MaximumFileSize > 0 && fi.Size() > *s.MaximumFileSize {
-  logrus.Debugf("\nyara: %v: Skipping large file, size=%v, max_size=%v", fileName, fi.Size(), *s.MaximumFileSize)
- return nil
+ matchesMetaData := make([]string, len(m.Metas))
+ for _, strMeta := range m.Metas {
+ matchesMetaData = append(matchesMetaData, fmt.Sprintf("%v : %v \n", strMeta.Identifier, strMeta.Value))
  }
- err = yrScanner.ScanFileDescriptor(f.Fd())
- if err != nil {
- fmt.Println("Scan File Descriptor error, trying alternative", err)
- var buf []byte
- if buf, err = io.ReadAll(f); err != nil {
- logrus.Errorf("yara: %s: Error reading file, error=%s",
- fileName, err.Error())
- return filepath.SkipDir
- }
- err = yrScanner.ScanMem(buf)
- if err != nil {
- fmt.Println("Scan File Mmory Error", err)
- return filepath.SkipDir
- }
 
- }
- var iocsFound []output.IOCFound
- totalMatchesStringData := make([]string, 0)
- for _, m := range matches {
- matchesStringData := make([]string, len(m.Strings))
- for _, str := range m.Strings {
- if !strings.Contains(strings.Join(matchesStringData, " "), string(str.Data)) {
- matchesStringData = append(matchesStringData, string(str.Data))
- totalMatchesStringData = append(totalMatchesStringData, string(str.Data))
- }
- }
- matchesMetaData := make([]string, len(m.Metas))
- for _, strMeta := range m.Metas {
- matchesMetaData = append(matchesMetaData, fmt.Sprintf("%v : %v \n", strMeta.Identifier, strMeta.Value))
+ iocsFound = append(iocsFound, output.IOCFound{
+ RuleName: m.Rule,
+ CategoryName: m.Tags,
+ StringsToMatch: matchesStringData,
+ Meta: matchesMetaData,
+ CompleteFilename: fileName,
+ })
+ }
+ var fileMat fileMatches
+ fileMat.fileName = fileName
+ fileMat.iocs = iocsFound
+ updatedSeverity, updatedScore := calculateSeverity(totalMatchesStringData, "low", 0)
+ fileMat.updatedSeverity = updatedSeverity
+ fileMat.updatedScore = updatedScore
+ // var isFirstIOC bool = true
+ if len(matches) > 0 {
+ // output.PrintColoredIOC(tempIOCsFound, &isFirstIOC, fileMat.updatedScore, fileMat.updatedSeverity)
+ for _, m := range iocsFound {
+ if isSharedLib {
+ m.FileSeverity = "low"
+ } else {
+ m.FileSeverity = updatedSeverity
  }
-
- iocsFound = append(iocsFound, output.IOCFound{
- RuleName: m.Rule,
- CategoryName: m.Tags,
- StringsToMatch: matchesStringData,
- Meta: matchesMetaData,
- CompleteFilename: fileName,
- })
- }
- var fileMat fileMatches
- fileMat.fileName = fileName
- fileMat.iocs = iocsFound
- updatedSeverity, updatedScore := calculateSeverity(totalMatchesStringData, "low", 0)
- fileMat.updatedSeverity = updatedSeverity
- fileMat.updatedScore = updatedScore
- // var isFirstIOC bool = true
- if len(matches) > 0 {
- // output.PrintColoredIOC(tempIOCsFound, &isFirstIOC, fileMat.updatedScore, fileMat.updatedSeverity)
- for _, m := range iocsFound {
- if isSharedLib {
- m.FileSeverity = "low"
- } else {
- m.FileSeverity = updatedSeverity
+ m.FileSevScore = updatedScore
+ StringsMatch := make([]string, 0)
+ for _, c := range m.StringsToMatch {
+ if len(c) > 0 {
+ StringsMatch = append(StringsMatch, c)
  }
-  m.FileSevScore = updatedScore
-  StringsMatch := make([]string, 0)
-  for _, c := range m.StringsToMatch {
-  if len(c) > 0 {
-  StringsMatch = append(StringsMatch, c)
-  }
-  }
- m.StringsToMatch = StringsMatch
- m.LayerID = layer
- summary := ""
- class := "Undefined"
- m.MetaRules = make(map[string]string)
- for _, c := range m.Meta {
- var metaSplit = strings.Split(c, " : ")
- if len(metaSplit) > 1 {
-
- // fmt.Fprintf(os.Stdout, Indent3+jsonMarshal(metaSplit[0])+":"+jsonMarshal(strings.Replace(metaSplit[1], "\n", "", -1))+",\n")
- m.MetaRules[metaSplit[0]] = strings.ReplaceAll(metaSplit[1], "\n", "")
- if metaSplit[0] == "description" {
- str := []string{"The file has a rule match that ", strings.ReplaceAll(metaSplit[1], "\n", "") + "."}
+ }
+ m.StringsToMatch = StringsMatch
+ m.LayerID = layer
+ summary := ""
+ class := "Undefined"
+ m.MetaRules = make(map[string]string)
+ for _, c := range m.Meta {
+ var metaSplit = strings.Split(c, " : ")
+ if len(metaSplit) > 1 {
+
+  // fmt.Fprintf(os.Stdout, Indent3+jsonMarshal(metaSplit[0])+":"+jsonMarshal(strings.Replace(metaSplit[1], "\n", "", -1))+",\n")
+  m.MetaRules[metaSplit[0]] = strings.ReplaceAll(metaSplit[1], "\n", "")
+  if metaSplit[0] == "description" {
+  str := []string{"The file has a rule match that ", strings.ReplaceAll(metaSplit[1], "\n", "") + "."}
+  summary += strings.Join(str, " ")
+ } else {
+ if metaSplit[0] == "info" {
+  class = strings.TrimSpace(strings.ReplaceAll(metaSplit[1], "\n", ""))
+ } else if len(metaSplit[0]) > 0 {
+ str := []string{"The matched rule file's ", metaSplit[0], " is", strings.ReplaceAll(metaSplit[1], "\n", "") + "."}
  summary += strings.Join(str, " ")
- } else {
- if metaSplit[0] == "info" {
- class = strings.TrimSpace(strings.ReplaceAll(metaSplit[1], "\n", ""))
- } else if len(metaSplit[0]) > 0 {
- str := []string{"The matched rule file's ", metaSplit[0], " is", strings.ReplaceAll(metaSplit[1], "\n", "") + "."}
- summary += strings.Join(str, " ")
- }
  }
  }
  }
- m.Summary = summary
- m.Class = class
- // *(*(*iocs)) = append(*(*(*iocs)), m)
- *iocs = append(*iocs, m)
  }
+ m.Summary = summary
+ m.Class = class
+ // *(*(*iocs)) = append(*(*(*iocs)), m)
+ *iocs = append(*iocs, m)
  }
-
  }
  return err
 }