Skip to content

Commit

Permalink
refactor: optimize control flow
Browse files Browse the repository at this point in the history
  • Loading branch information
@pluveto
pluveto committed Feb 8, 2022
1 parent 68cf613 commit d79e6e2
Show file tree
Hide file tree
Showing 7 changed files with 106 additions and 95 deletions.
1 change: 1 addition & 0 deletions lib/model/task.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,6 @@ type Task struct {
Ignored bool `toml:"ignored" mapstructure:"ignored"`
RawUrl string `toml:"raw_url" mapstructure:"raw_url"`
Url string `toml:"url" mapstructure:"url"`
CreateTime time.Time `toml:"create_time" mapstructure:"create_time"`
FinishTime time.Time `toml:"finish_time" mapstructure:"finish_time"`
}
12 changes: 9 additions & 3 deletions lib/result/result.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,13 @@ func (r Result[T]) Ok() bool {
return r.Err == nil
}

func FromGoRet[T any](in ...interface{}) Result[T] {
// From
// In Golang we usually return by the format of `data ,err`.
// This function convert it to a Result[T], so that you can handle error in a tidy way
func From[T any](in ...interface{}) Result[T] {
if len(in) != 2 {
panic("unexpected number of return values")
}
if nil != in[1] {
return Result[T]{
Err: in[1].(error),
Expand All @@ -34,9 +40,9 @@ func (r Result[T]) ValueOrPanic() T {
panic(r.Err)
}

type ErrorHandler func(err error)
type AbortHandler func(err error)

var AbortErr ErrorHandler
var AbortErr AbortHandler

func (r Result[T]) ValueOrExit() T {
if r.Err == nil {
Expand Down
11 changes: 0 additions & 11 deletions lib/upyun/sdk.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -415,17 +415,6 @@ func StringMd5(s string) string {
return fmt.Sprintf("%x", h.Sum(nil))
}

func FileMd5(name string) string {
f, err := os.Open(name)
if err != nil {
return ""
}
defer f.Close()

h := md5.New()
io.Copy(h, f)
return fmt.Sprintf("%x", h.Sum(nil))
}

func timeoutDialer(timeout int) func(string, string) (net.Conn, error) {
return func(netw, addr string) (c net.Conn, err error) {
Expand Down
1 change: 0 additions & 1 deletion lib/upyun/uploader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,6 @@ func (u UpyunUploader) Upload(t *model.Task) error {

func (u *UpyunUploader) buildUrl(urlfmt, path string) string {
r := strings.NewReplacer(
// <BucketName-APPID>.cos.<Region>.myqcloud.com
"{host}", u.Config.Host,
"{path}", path,
)
Expand Down
3 changes: 2 additions & 1 deletion lib/xmap/xmap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ func GetDeep[T any](m map[string]interface{}, path string) (ret T, err error) {
if key == "" {
continue
}
// match xxx[i] $1 $2
// match xxx[i] (array form)
// $1 $2
r := regexp.MustCompile(`^(.*)\[(\d+)\]$`)
arrIndex := 0
matches := r.FindAllString(key, -1)
Expand Down
96 changes: 53 additions & 43 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,39 +41,12 @@ func main() {

func mainCommand() {
// parse cli args
arg.MustParse(&xapp.AppOpt)
xapp.AppOpt.TargetDir = strings.Trim(xapp.AppOpt.TargetDir, "/")
xapp.AppOpt.ApplicationPath = strings.Trim(xapp.AppOpt.ApplicationPath, "/")
if len(xapp.AppOpt.ApplicationPath) > 0 {
xpath.ApplicationPath = xapp.AppOpt.ApplicationPath
}
if xapp.AppOpt.SizeLimit != nil && *xapp.AppOpt.SizeLimit >= 0 {
xapp.MaxUploadSize = *xapp.AppOpt.SizeLimit
}
if false == xapp.AppOpt.NoLog {
xlog.GVerbose.LogEnabled = true
xlog.GVerbose.LogFile = xpath.MustGetApplicationPath("upgit.log")
xlog.GVerbose.LogFileMaxSize = 2 * 1024 * 1024 // 2MiB
xlog.GVerbose.Info("Started")
xlog.GVerbose.TruncatLog()
}
xlog.GVerbose.VerboseEnabled = xapp.AppOpt.Verbose
xlog.GVerbose.TraceStruct(xapp.AppOpt)
loadCliOpts()

// load config
loadEnvConfig(&xapp.AppCfg)
loadTomlConfig(&xapp.AppCfg)

// fill config
xapp.AppCfg.Rename = strings.Trim(xapp.AppCfg.Rename, "/")
xapp.AppCfg.Rename = xstrings.RemoveFmtUnderscore(xapp.AppCfg.Rename)
loadConfig(&xapp.AppCfg)

// -- integrated formats
if nil == xapp.AppCfg.OutputFormats {
xapp.AppCfg.OutputFormats = make(map[string]string)
}
xapp.AppCfg.OutputFormats["markdown"] = `![{url_fname}]({url})`
xapp.AppCfg.OutputFormats["url"] = `{url}`
xlog.GVerbose.TraceStruct(xapp.AppCfg)

// handle clipboard if need
Expand All @@ -83,7 +56,7 @@ func mainCommand() {
validArgs()

// executing uploading
upload()
dispatchUploader()

if xapp.AppOpt.Wait {
fmt.Scanln()
Expand All @@ -92,6 +65,27 @@ func mainCommand() {
return
}

func loadCliOpts() {
arg.MustParse(&xapp.AppOpt)
xapp.AppOpt.TargetDir = strings.Trim(xapp.AppOpt.TargetDir, "/")
xapp.AppOpt.ApplicationPath = strings.Trim(xapp.AppOpt.ApplicationPath, "/")
if len(xapp.AppOpt.ApplicationPath) > 0 {
xpath.ApplicationPath = xapp.AppOpt.ApplicationPath
}
if xapp.AppOpt.SizeLimit != nil && *xapp.AppOpt.SizeLimit >= 0 {
xapp.MaxUploadSize = *xapp.AppOpt.SizeLimit
}
if false == xapp.AppOpt.NoLog {
xlog.GVerbose.LogEnabled = true
xlog.GVerbose.LogFile = xpath.MustGetApplicationPath("upgit.log")
xlog.GVerbose.LogFileMaxSize = 2 * 1024 * 1024 // 2MiB
xlog.GVerbose.Info("Started")
xlog.GVerbose.TruncatLog()
}
xlog.GVerbose.VerboseEnabled = xapp.AppOpt.Verbose
xlog.GVerbose.TraceStruct(xapp.AppOpt)
}

func onUploaded(r result.Result[*model.Task]) {
if !r.Ok() && xapp.AppOpt.OutputType == xapp.O_Stdout {
fmt.Println("Failed: " + r.Err.Error())
Expand Down Expand Up @@ -180,7 +174,7 @@ func validArgs() {
}
}

func loadTomlConfig(cfg *xapp.Config) {
func loadConfig(cfg *xapp.Config) {

homeDir, err := os.UserHomeDir()
xlog.AbortErr(err)
Expand All @@ -205,13 +199,20 @@ func loadTomlConfig(cfg *xapp.Config) {
xlog.AbortErr(fmt.Errorf("invalid config: " + err.Error()))
}
xapp.ConfigFilePath = configFile
return
break
}

}
// fill config
xapp.AppCfg.Rename = strings.Trim(xapp.AppCfg.Rename, "/")
xapp.AppCfg.Rename = xstrings.RemoveFmtUnderscore(xapp.AppCfg.Rename)

// -- integrated formats
if nil == xapp.AppCfg.OutputFormats {
xapp.AppCfg.OutputFormats = make(map[string]string)
}
xapp.AppCfg.OutputFormats["markdown"] = `![{url_fname}]({url})`
xapp.AppCfg.OutputFormats["url"] = `{url}`

type Nullable[T any] struct {
Value *T
}

// UploadAll will upload all given file to targetDir.
Expand All @@ -227,20 +228,28 @@ func UploadAll(uploader model.Uploader, localPaths []string, targetDir string) {
TargetDir: targetDir,
RawUrl: "",
Url: "",
FinishTime: time.Now(),
CreateTime: time.Now(),
}
var err error
// ignore non-local path
if strings.HasPrefix(localPath, "http") {
task.Ignored = true
task.Status = model.TASK_FINISHED
} else {
uploader.Upload(&task)
err = uploader.Upload(&task)
}
ret = result.Result[*model.Task]{
Value: &task,
if err != nil {
task.Status = model.TASK_FAILED
ret = result.Result[*model.Task]{
Err: err,
}
} else {
ret = result.Result[*model.Task]{
Value: &task,
}
}

if ret.Err == nil {
if err == nil {
xlog.GVerbose.TraceStruct(ret.Value)
}
callback := uploader.GetCallback()
Expand All @@ -250,7 +259,7 @@ func UploadAll(uploader model.Uploader, localPaths []string, targetDir string) {
}
}

func upload() {
func dispatchUploader() {
uploaderId := xstrings.ValueOrDefault(xapp.AppOpt.Uploader, xapp.AppCfg.DefaultUploader)
xlog.GVerbose.Info("uploader: " + uploaderId)
if uploaderId == "github" {
Expand Down Expand Up @@ -304,10 +313,10 @@ func upload() {
// load file to json
uploaderDef, err := xext.GetExtDefinitionInterface(extDir, fname)
xlog.AbortErr(err)
if result.FromGoRet[string](xmap.GetDeep[string](uploaderDef, `meta.id`)).ValueOrExit() != uploaderId {
if result.From[string](xmap.GetDeep[string](uploaderDef, `meta.id`)).ValueOrExit() != uploaderId {
continue
}
if result.FromGoRet[string](xmap.GetDeep[string](uploaderDef, "meta.type")).ValueOrExit() != "simple-http-uploader" {
if result.From[string](xmap.GetDeep[string](uploaderDef, "meta.type")).ValueOrExit() != "simple-http-uploader" {
continue
}
uploader = &SimpleHttpUploader{OnTaskStatusChanged: onUploaded, Definition: uploaderDef}
Expand Down Expand Up @@ -367,6 +376,7 @@ func loadEnvConfig(cfg *xapp.Config) {
}

func loadGithubUploaderEnvConfig(gCfg *GithubUploaderConfig) {
// TODO: Auto generate env key name and adapt for all uploaders
if pat, found := syscall.Getenv("GITHUB_TOKEN"); found {
gCfg.PAT = pat
}
Expand Down
77 changes: 41 additions & 36 deletions simple_http_uploader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,11 +150,11 @@ func GetValueByConfigTag(data interface{}, key string) (ret interface{}) {

func (u SimpleHttpUploader) UploadFile(task *model.Task) (rawUrl string, err error) {
// == prepare method and url ==
method := result.FromGoRet[string](xmap.GetDeep[string](u.Definition, "http.request.method")).ValueOrExit()
urlRaw := result.FromGoRet[string](xmap.GetDeep[string](u.Definition, "http.request.url")).ValueOrExit()
params := result.FromGoRet[map[string]interface{}](xmap.GetDeep[map[string]interface{}](u.Definition, "http.request.params")).ValueOrDefault(map[string]interface{}{})
method := result.From[string](xmap.GetDeep[string](u.Definition, "http.request.method")).ValueOrExit()
urlRaw := result.From[string](xmap.GetDeep[string](u.Definition, "http.request.url")).ValueOrExit()
params := result.From[map[string]interface{}](xmap.GetDeep[map[string]interface{}](u.Definition, "http.request.params")).ValueOrDefault(map[string]interface{}{})
u.replaceDictPlaceholder(params, *task)
url := result.FromGoRet[*url.URL](url.Parse(u.replaceStringPlaceholder(urlRaw, *task))).ValueOrExit()
url := result.From[*url.URL](url.Parse(u.replaceStringPlaceholder(urlRaw, *task))).ValueOrExit()
query := url.Query()
for paramName, paramValue := range params {
query.Add(paramName, paramValue.(string))
Expand All @@ -163,50 +163,50 @@ func (u SimpleHttpUploader) UploadFile(task *model.Task) (rawUrl string, err err
xlog.GVerbose.Info("Method: %s, URL: %s", method, url.String())

// == Prepare header ==
headers := result.FromGoRet[map[string]interface{}](xmap.GetDeep[map[string]interface{}](u.Definition, "http.request.headers")).ValueOrExit()
u.replaceDictPlaceholder(headers, *task)
defHeaders := result.From[map[string]interface{}](xmap.GetDeep[map[string]interface{}](u.Definition, "http.request.headers")).ValueOrExit()
u.replaceDictPlaceholder(defHeaders, *task)

xlog.GVerbose.Trace("unformatted headers:")
xlog.GVerbose.TraceStruct(headers)
headerCache := make(http.Header)
for k, v := range headers {
headerCache.Set(k, u.replaceStringPlaceholder(v.(string), *task))
xlog.GVerbose.TraceStruct(defHeaders)
header := make(http.Header)
for k, v := range defHeaders {
header.Set(k, u.replaceStringPlaceholder(v.(string), *task))
}
if headerCache.Get("Content-Type") == "" {
headerCache.Set("Content-Type", "application/octet-stream")
if header.Get("Content-Type") == "" {
header.Set("Content-Type", "application/octet-stream")
}
xlog.GVerbose.Trace("formatted headers:")
xlog.GVerbose.TraceStruct(map[string][]string(headerCache))
xlog.GVerbose.TraceStruct(map[string][]string(header))
// upload file according to content-type
contentType := headerCache.Get("Content-Type")

// == Prepare body ==
var body io.ReadCloser
if contentType == "application/octet-stream" {
body = ioutil.NopCloser(bytes.NewReader(result.FromGoRet[[]byte](ioutil.ReadFile(task.LocalPath)).ValueOrExit()))
} else if contentType == "multipart/form-data" {
body = u.buildMultipartFormData(task, &headerCache)
switch header.Get("Content-Type") {
case "application/octet-stream":
body = ioutil.NopCloser(bytes.NewReader(result.From[[]byte](ioutil.ReadFile(task.LocalPath)).ValueOrExit()))

case "multipart/form-data":
body = u.buildMultipartFormData(task, &header)
}

// == Create Request ==
req := result.FromGoRet[*http.Request](http.NewRequest(method, u.replaceStringPlaceholder(url.String(), *task), body)).ValueOrExit()
req.Header = headerCache
req := result.From[*http.Request](http.NewRequest(method, u.replaceStringPlaceholder(url.String(), *task), body)).ValueOrExit()
req.Header = header
xlog.GVerbose.Trace("do headers:")
xlog.GVerbose.TraceStruct(map[string][]string(req.Header))

// == Do Request ==
resp := result.FromGoRet[*http.Response](http.DefaultClient.Do(req)).ValueOrExit()
bodyBytes := result.FromGoRet[[]byte](ioutil.ReadAll(resp.Body)).ValueOrExit()
resp := result.From[*http.Response](http.DefaultClient.Do(req)).ValueOrExit()
bodyBytes := result.From[[]byte](ioutil.ReadAll(resp.Body)).ValueOrExit()
xlog.GVerbose.Info("response body:" + string(bodyBytes))
// check statuscode
if !(200 <= resp.StatusCode && resp.StatusCode < 300) {
return "", fmt.Errorf("unexpected status code %d. response: %s", resp.StatusCode, string(bodyBytes))
}
// == Construct rawUrl from Response ==
urlFrom := result.FromGoRet[string](xmap.GetDeep[string](u.Definition, "upload.rawUrl.from")).ValueOrExit()
if urlFrom == "json_response" {
// read response body json

urlFrom := result.From[string](xmap.GetDeep[string](u.Definition, "upload.rawUrl.from")).ValueOrExit()
switch urlFrom {
case "json_response":
var respJson map[string]interface{}
err := json.Unmarshal(bodyBytes, &respJson)
if err != nil {
Expand All @@ -215,7 +215,7 @@ func (u SimpleHttpUploader) UploadFile(task *model.Task) (rawUrl string, err err
if !(200 <= resp.StatusCode && resp.StatusCode < 300) {
return "", fmt.Errorf("response status code %d is not expected. resp: %s", resp.StatusCode, string(bodyBytes))
}
rawUrlPath := result.FromGoRet[string](xmap.GetDeep[string](u.Definition, "upload.rawUrl.path")).ValueOrExit()
rawUrlPath := result.From[string](xmap.GetDeep[string](u.Definition, "upload.rawUrl.path")).ValueOrExit()
rawUrl, err = xmap.GetDeep[string](respJson, rawUrlPath)
if err != nil {
return "", errors.New("rawUrl path is not valid: " + err.Error())
Expand All @@ -224,16 +224,21 @@ func (u SimpleHttpUploader) UploadFile(task *model.Task) (rawUrl string, err err
return "", fmt.Errorf("unable to get url. resp: %s", string(bodyBytes))
}
xlog.GVerbose.Trace("got rawUrl from resp: " + rawUrl)
} else if urlFrom == "text_response" {

case "text_response":
rawUrl = string(bodyBytes)
} else if urlFrom == "template" {
template := result.FromGoRet[string](xmap.GetDeep[string](u.Definition, "upload.rawUrl.template")).ValueOrExit()

case "template":
template := result.From[string](xmap.GetDeep[string](u.Definition, "upload.rawUrl.template")).ValueOrExit()
rawUrl = u.replaceStringPlaceholder(template, *task)
} else if urlFrom == "response_header" {

case "response_header":

// read response header
key := result.FromGoRet[string](xmap.GetDeep[string](u.Definition, "upload.rawUrl.header")).ValueOrExit()
key := result.From[string](xmap.GetDeep[string](u.Definition, "upload.rawUrl.header")).ValueOrExit()
rawUrl = resp.Header.Get(key)
} else {

default:
return "", errors.New("unsupported rawUrl source" + urlFrom)
}
return
Expand All @@ -242,7 +247,7 @@ func (u SimpleHttpUploader) UploadFile(task *model.Task) (rawUrl string, err err
func (u SimpleHttpUploader) buildMultipartFormData(task *model.Task, headerCache *http.Header) (body io.ReadCloser) {
var bodyBuff bytes.Buffer
mulWriter := multipart.NewWriter(&bodyBuff)
bodyTpl := result.FromGoRet[map[string]interface{}](xmap.GetDeep[map[string]interface{}](u.Definition, "http.request.body")).ValueOrExit()
bodyTpl := result.From[map[string]interface{}](xmap.GetDeep[map[string]interface{}](u.Definition, "http.request.body")).ValueOrExit()
for fieldName, fieldMeta_ := range bodyTpl {
xlog.GVerbose.Trace("processing field: " + fieldName)
fieldMeta := fieldMeta_.(map[string]interface{})
Expand All @@ -256,8 +261,8 @@ func (u SimpleHttpUploader) buildMultipartFormData(task *model.Task, headerCache

} else if fieldType == "file" {
fileName := filepath.Base(task.LocalPath)
part := result.FromGoRet[io.Writer](mulWriter.CreateFormFile(fieldName, fileName)).ValueOrExit()
n, err := part.Write(result.FromGoRet[[]byte](ioutil.ReadFile(task.LocalPath)).ValueOrExit())
part := result.From[io.Writer](mulWriter.CreateFormFile(fieldName, fileName)).ValueOrExit()
n, err := part.Write(result.From[[]byte](ioutil.ReadFile(task.LocalPath)).ValueOrExit())
xlog.AbortErr(err)
xlog.GVerbose.Trace("field(file) value: "+"[file (len=%d, name=%s)]", n, fileName)

Expand Down

0 comments on commit d79e6e2

Please sign in to comment.