Merge branch 'archive'

gjolly · Aug 13, 2022 · eba01a4 · eba01a4
2 parents e21ea6f + 2c12bb4
commit eba01a4
Show file tree

Hide file tree

Showing 5 changed files with 714 additions and 172 deletions.
diff --git a/cmd/rmadison-server/main.go b/cmd/rmadison-server/main.go
@@ -1,182 +1,21 @@
 package main
 
 import (
-	"compress/gzip"
 	"encoding/json"
-	"fmt"
-	"io/ioutil"
+	"flag"
 	"log"
 	"net/http"
+	"net/url"
+	"os"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/gjolly/go-rmadisson/pkg/debian"
 	"github.com/go-resty/resty/v2"
 )
 
-func downloadRawIndex(client *resty.Client, series, pocket, component, arch string) (string, error) {
-	archive := "https://archive.ubuntu.com/ubuntu"
-	if arch != "amd64" && arch != "i386" {
-		archive = "https://ports.ubuntu.com/ubuntu-ports"
-	}
-	url := fmt.Sprintf(
-		"%v/dists/%v%v/%v/binary-%v/Packages.gz",
-		archive,
-		series,
-		pocket,
-		component,
-		arch,
-	)
-	resp, err := client.R().
-		SetDoNotParseResponse(true).
-		Get(url)
-	if err != nil {
-		return "", err
-	}
-
-	body := resp.RawBody()
-	gzipReader, err := gzip.NewReader(body)
-	if err != nil {
-		return "", err
-	}
-	defer gzipReader.Close()
-
-	result, err := ioutil.ReadAll(gzipReader)
-	if err != nil {
-		return "", err
-	}
-
-	return fmt.Sprintf("%s", result), nil
-}
-
-func getIndex(client *resty.Client, pkgInfoChan chan *debian.PackageInfo, series, pocket, component, arch string) error {
-	rawIndex, err := downloadRawIndex(client, series, pocket, component, arch)
-	if err != nil {
-		return err
-	}
-
-	packageInfo := strings.Split(rawIndex, "\n\n")
-
-	for _, info := range packageInfo {
-		infoLines := strings.Split(info, "\n")
-		pkgName := ""
-
-		var pkgInfo *debian.PackageInfo
-
-		for _, line := range infoLines {
-			cleanLine := strings.Trim(line, "\n")
-			if cleanLine == "" {
-				continue
-			}
-
-			rawAttribute := strings.Split(cleanLine, ": ")
-
-			if len(rawAttribute) < 2 {
-				//fmt.Printf("failed to parse property: %#v\n", rawAttribute)
-				continue
-			}
-			key := rawAttribute[0]
-			value := strings.TrimPrefix(cleanLine, fmt.Sprintf("%v: ", key))
-
-			// here we assume that we see Package before any other field
-			// it makes sense but it's also not very safe
-			if key == "Package" {
-				pkgName = value
-				pkgInfo = &debian.PackageInfo{
-					Name:         pkgName,
-					Component:    component,
-					Suite:        series,
-					Pocket:       pocket,
-					Architecture: arch,
-				}
-			}
-			if key == "Version" {
-				pkgInfo.Version = value
-			}
-		}
-
-		if pkgInfo != nil {
-			pkgInfoChan <- pkgInfo
-		}
-	}
-
-	return nil
-}
-
-// Cache is a key value store
-type Cache struct {
-	Packages map[string][]*debian.PackageInfo
-}
-
-func (c *Cache) updateCache() {
-	client := resty.New()
-	client.SetRetryCount(3).
-		SetRetryWaitTime(5 * time.Second).
-		SetRetryMaxWaitTime(20 * time.Second)
-
-	pockets := []string{"", "-updates", "-proposed"}
-	components := []string{"main", "universe", "multiverse"}
-	series := []string{"bionic", "focal", "impish", "jammy", "kinetic"}
-	archs := []string{"amd64", "arm64"}
-
-	resultChan := make(chan *debian.PackageInfo, 1000)
-	done := make(chan struct{})
-	wg := new(sync.WaitGroup)
-
-	for _, suite := range series {
-		for _, component := range components {
-			for _, pocket := range pockets {
-				for _, arch := range archs {
-					wg.Add(1)
-					go func(suite, pocket, component, arch string) {
-						defer wg.Done()
-						err := getIndex(client, resultChan, suite, pocket, component, arch)
-						if err != nil {
-							log.Println(err)
-							return
-						}
-					}(suite, pocket, component, arch)
-				}
-			}
-		}
-	}
-
-	go func() {
-		wg.Wait()
-		close(done)
-	}()
-
-	for {
-		select {
-		case info := <-resultChan:
-			if _, exist := c.Packages[info.Name]; !exist {
-				c.Packages[info.Name] = []*debian.PackageInfo{info}
-				continue
-			}
-
-			updated := false
-			for _, currentInfo := range c.Packages[info.Name] {
-				if currentInfo.Component == info.Component && currentInfo.Suite == info.Suite && currentInfo.Pocket == info.Pocket && currentInfo.Architecture == info.Architecture {
-					currentInfo.Version = info.Version
-
-					updated = true
-					break
-				}
-			}
-			if !updated {
-				c.Packages[info.Name] = append(c.Packages[info.Name], info)
-			}
-		case <-done:
-			if len(resultChan) == 0 {
-				return
-			}
-		}
-	}
-}
-
 type httpHandler struct {
-	Cache *Cache
+	Cache *debian.Archive
 }
 
 func (h httpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -188,13 +27,21 @@ func (h httpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	info, ok := h.Cache.Packages[pkg]
+	allInfo, ok := h.Cache.Packages[pkg]
 	if !ok {
 		w.WriteHeader(http.StatusNotFound)
 		return
 	}
 
-	jsonInfo, err := json.Marshal(info)
+	// convert the dictionary to a list
+	fmtInfo := make([]*debian.PackageInfo, len(allInfo))
+	i := 0
+	for _, info := range allInfo {
+		fmtInfo[i] = info
+		i++
+	}
+
+	jsonInfo, err := json.Marshal(fmtInfo)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		return
@@ -205,14 +52,47 @@ func (h httpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 }
 
 func main() {
-	cache := new(Cache)
-	cache.Packages = make(map[string][]*debian.PackageInfo)
+	flag.Parse()
+	cacheDir := flag.Arg(0)
+	if cacheDir == "" {
+		cacheDir, _ = os.MkdirTemp("", "gormadisontest")
+	}
+
+	baseURL, _ := url.Parse("https://archive.ubuntu.com/ubuntu/dists")
+	portsURL, _ := url.Parse("https://ports.ubuntu.com/dists")
+
+	client := resty.New()
+	cache := &debian.Archive{
+		BaseURL:  *baseURL,
+		PortsURL: *portsURL,
+		Client:   client,
+		Pockets: []string{
+			"bionic", "bionic-updates",
+			"focal", "focal-updates",
+			"jammy", "jammy-updates",
+			"kinetic",
+		},
+		CacheDir: cacheDir,
+	}
+
+	packages, err := cache.InitCache()
+	if err != nil {
+		log.Println("error reading existing cache data", err)
+	}
+	log.Println("packages in cache:", packages)
 
 	go func() {
-		t := time.NewTicker(15 * time.Minute)
+		t := time.NewTicker(5 * time.Minute)
 		for {
-			cache.updateCache()
-			log.Println("cache updated")
+			now := time.Now()
+			_, pkgStats, err := cache.RefreshCache()
+			duration := time.Now().Sub(now)
+			if err != nil {
+				log.Printf("cache refreshed (with error '%v') in %v, %v packages in db", err, duration.Seconds(), pkgStats)
+			} else {
+				log.Printf("cache refreshed in %v, %v packages in db", duration.Seconds(), pkgStats)
+			}
+
 			<-t.C
 		}
 	}()

diff --git a/go.mod b/go.mod
@@ -6,5 +6,6 @@ require (
 	github.com/go-resty/resty/v2 v2.7.0 // indirect
 	github.com/klauspost/compress v1.15.6 // indirect
 	github.com/klauspost/pgzip v1.2.5 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	golang.org/x/net v0.0.0-20211029224645-99673261e6eb // indirect
 )
diff --git a/go.sum b/go.sum
@@ -4,6 +4,8 @@ github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRs
 github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
 github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
 github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 golang.org/x/net v0.0.0-20211029224645-99673261e6eb h1:pirldcYWx7rx7kE5r+9WsOXPXK0+WH5+uZ7uPmJ44uM=
 golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=