电影显示同一网站不同域名合并

controllers/api/admin.go

@@ -609,7 +609,7 @@ func (c *AdminAPIController) MovieUpdate() {
  return
  }
  movie := new(Movie)
- movie.Id = movurl.Movie.Id
+ movie.Id = movurl.MovieId
  movie.Name = name
  _, err = movie.Update([]string{"name"})
  if nil != err {

go.mod

@@ -18,52 +18,13 @@ require (
 )
 
 require (
- github.com/beego/bee/v2 v2.1.0 // indirect
- github.com/cilium/ebpf v0.12.3 // indirect
- github.com/cosiner/argv v0.1.0 // indirect
  github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
- github.com/derekparker/trie v0.0.0-20230829180723-39f4de51ef7d // indirect
- github.com/flosch/pongo2 v0.0.0-20200913210552-0d938eb266f3 // indirect
- github.com/flosch/pongo2/v6 v6.0.0 // indirect
  github.com/fsnotify/fsnotify v1.7.0 // indirect
- github.com/go-delve/delve v1.21.2 // indirect
- github.com/go-delve/liner v1.2.3-0.20220127212407-d32d89dd2a5d // indirect
- github.com/google/go-dap v0.11.0 // indirect
- github.com/gorilla/websocket v1.5.1 // indirect
- github.com/hashicorp/hcl v1.0.0 // indirect
- github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
  github.com/lib/pq v1.10.9 // indirect
- github.com/magiconair/properties v1.8.7 // indirect
- github.com/mattn/go-colorable v0.1.13 // indirect
- github.com/mattn/go-isatty v0.0.20 // indirect
- github.com/mattn/go-runewidth v0.0.15 // indirect
  github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
- github.com/pelletier/go-toml v1.9.5 // indirect
- github.com/pelletier/go-toml/v2 v2.1.1 // indirect
  github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
- github.com/rivo/uniseg v0.4.4 // indirect
- github.com/sagikazarmark/locafero v0.4.0 // indirect
- github.com/sagikazarmark/slog-shim v0.1.0 // indirect
- github.com/sirupsen/logrus v1.9.3 // indirect
- github.com/smartwalle/pongo2render v1.0.2 // indirect
- github.com/sourcegraph/conc v0.3.0 // indirect
- github.com/spf13/afero v1.11.0 // indirect
- github.com/spf13/cast v1.6.0 // indirect
- github.com/spf13/jwalterweatherman v1.1.0 // indirect
- github.com/spf13/pflag v1.0.5 // indirect
- github.com/spf13/viper v1.18.2 // indirect
  github.com/stretchr/testify v1.8.4 // indirect
- github.com/subosito/gotenv v1.6.0 // indirect
  github.com/valyala/bytebufferpool v1.0.0 // indirect
- go.starlark.net v0.0.0-20231121155337-90ade8b19d09 // indirect
- go.uber.org/atomic v1.11.0 // indirect
- go.uber.org/multierr v1.11.0 // indirect
- golang.org/x/arch v0.6.0 // indirect
- golang.org/x/exp v0.0.0-20231226003508-02704c960a9b // indirect
- golang.org/x/mod v0.14.0 // indirect
- golang.org/x/tools v0.16.1 // indirect
- gopkg.in/ini.v1 v1.67.0 // indirect
- gopkg.in/yaml.v2 v2.4.0 // indirect
 )
 
 require (

models/job-count-model.go

@@ -8,7 +8,7 @@ import (
  "github.com/beego/beego/v2/client/orm"
 )
 
-//JobCount 添加tag标签
+// JobCount 添加tag标签
 type JobCount struct {
  BaseModel
  JobTitle string `json:"job_title"`
@@ -20,7 +20,7 @@ func init() {
  orm.RegisterModel(new(JobCount))
 }
 
-//Save 添加记录
+// Save 添加记录
 func (jobCount *JobCount) Save() (int64, error) {
  o := orm.NewOrm()
  return o.Insert(jobCount)
@@ -32,7 +32,7 @@ type JobData struct {
  Nodejs string `json:"nodejs"`
 }
 
-//GetCountData 获取记录列表，利用行专列将同一天数据一行展示
+// GetCountData 获取记录列表，利用行专列将同一天数据一行展示
 func (jobCount *JobCount) GetCountData(startDate, endDate string) ([]JobData, error) {
  o := orm.NewOrm()
  var jobCounts []JobData
@@ -50,7 +50,7 @@ func (jobCount *JobCount) GetCountData(startDate, endDate string) ([]JobData, er
  return jobCounts, err
 }
 
-//GetCountDataAll 所有历史数据，按月平均值统计
+// GetCountDataAll 所有历史数据，按月平均值统计
 func (jobCount *JobCount) GetCountDataAll() ([]JobData, error) {
  o := orm.NewOrm()
  var jobCounts []JobData
@@ -65,3 +65,20 @@ func (jobCount *JobCount) GetCountDataAll() ([]JobData, error) {
  logs.Info("每月平均查询到", num, "条数据")
  return jobCounts, err
 }
+
+type JobDataLanguage struct {
+ Count int //数量
+ Region string //地区
+ Language string //语言
+}
+
+// 保存数据
+func SaveJob(c JobDataLanguage) {
+ jobCount := &JobCount{
+ JobTitle: c.Language,
+ Amount: uint(c.Count),
+ Region: c.Region,
+ }
+ jobCount.Save()
+ logs.Info("保存job成功", c.Language, c.Count)
+}

models/movie-model.go

@@ -1,43 +1,33 @@
 package models
 
 import (
- "strconv"
  "sync"
 
  "github.com/beego/beego/v2/core/logs"
 
  "github.com/beego/beego/v2/client/orm"
 )
 
-// type Movie struct {
-// BaseModel
-// Name string `json:"name" orm:"size(50)"`
-// URL string `json:"urls" orm:"column(url)"`
-// Status int `json:"status"` //0:正常，1:禁用，2:有异常
-// StatusComment string `json:"statusCommnet"`
-// }
-
 type Movie struct {
  BaseModel
  Name string `json:"name" orm:"size(50)"`
- URLs []*MovieUrl `json:"urls" orm:"reverse(many)"`
- Status int `json:"status"` //0:正常，1:禁用，2:有异常
- StatusComment string `json:"statusCommnet"`
+ URLs []*MovieUrl `json:"urls" orm:"-"`
+ Status int `json:"-"`
+ StatusComment string `json:"-"`
 }
 type MovieUrl struct {
  BaseModel
- Movie  *Movie `orm:"rel(fk)"` // RelForeignKey relation
+ MovieId uint `json:"-" orm:"column(movie_id)"`
  URL string `json:"url" orm:"column(url)"`
  Status int `json:"status"` //0:正常，1:禁用，2:有异常
  StatusComment string `json:"statusCommnet"`
 }
 
-type MovieView struct {
- BaseModel
- Name string `json:"name" orm:"size(50)"`
- URL string `json:"url" orm:"column(url)"`
- Status int `json:"status"` //0:正常，1:禁用，2:有异常
- StatusComment string `json:"statusCommnet"`
+// 视图结构体
+type MovieVO struct {
+ Id uint `json:"id"`
+ Name string `json:"name"`
+ Movies []MovieUrl
 }
 
 func init() {
@@ -66,7 +56,7 @@ func (mov *Movie) MovieInfo() error {
  return nil
 }
 
-func (mov *Movie) GetMovieList(name, url string, start, perPage int) (list []MovieView, total int64, err error) {
+func (mov *Movie) GetMovieList(name, url string, start, perPage int) (list []Movie, total int64, err error) {
  o := orm.NewOrm()
  var condition = " WHERE 1 = 1 "
  if name != "" {
@@ -79,21 +69,46 @@ func (mov *Movie) GetMovieList(name, url string, start, perPage int) (list []Mov
  wg.Add(2)
  go func() {
  defer wg.Done()
- var sql = "SELECT movie_url.id, movie.name, movie_url.url, movie_url.status, movie_url.status_comment, movie_url.created_at,movie_url.updated_at FROM movie left join movie_url on movie.id = movie_url.movie_id "
+ var sql = "SELECT movie.id, movie.name FROM movie "
  sql += condition
- sql += " order by movie_url.status, movie.created_at LIMIT ?, ?"
- _, err1 := o.Raw(sql, strconv.Itoa(start), strconv.Itoa(perPage)).QueryRows(&list)
+ sql += " order by movie.created_at LIMIT ?, ?"
+ _, err1 := o.Raw(sql, start, perPage).QueryRows(&list)
  if err1 != nil {
  err = err1
  }
- // for idx, v := range list {
- // o.Raw("select * from movie_url where movie_id = ?", v.Id).QueryRows(&v.URLs)
- // list[idx].URLs = v.URLs
- // }
+ movieIds := []uint{}
+ for _, v := range list {
+ movieIds = append(movieIds, v.Id)
+ }
+ list2 := []MovieUrl{}
+ qs := "select * from movie_url where movie_id IN ("
+ for i, _ := range movieIds {
+ if i > 0 {
+ qs += ","
+ }
+ qs += "?"
+ }
+ qs += ")"
+ if _, err1 := o.Raw(qs, movieIds).QueryRows(&list2); err != nil {
+ err = err1
+ }
+ movieMap := make(map[uint][]*MovieUrl)
+ for i := 0; i < len(list2); i++ {
+ movieMap[list2[i].MovieId] = append(movieMap[list2[i].MovieId], &list2[i])
+ }
+ for idx, v := range list {
+ list[idx].Id = v.Id
+ list[idx].Name = v.Name
+ if _, ok := movieMap[v.Id]; ok {
+ list[idx].URLs = movieMap[v.Id]
+ } else {
+ list[idx].URLs = []*MovieUrl{}
+ }
+ }
  }()
  go func() {
  defer wg.Done()
- var sql = "SELECT COUNT(0) FROM movie left join movie_url on movie.id = movie_url.movie_id "
+ var sql = "SELECT COUNT(0) FROM movie "
  sql += condition
  err1 := o.Raw(sql).QueryRow(&total)
  if err1 != nil {

tests/default_test.go

@@ -137,7 +137,7 @@ func TestHtmlToText(t *testing.T) {
 }
 
 func TestRequestByAjax3(t *testing.T) {
- ch1 := make(chan util.JobDataLanguage)
+ ch1 := make(chan models.JobDataLanguage)
  go util.RequestByAjax3(ch1, "上海", "nodejs")
  t.Log(">>>>ch1", <-ch1)
 }

util/crawl.go

@@ -4,7 +4,7 @@ import (
  "encoding/json"
  "fmt"
  "io"
- . "metal/models"
+ "metal/models"
  "net/http"
  "net/url"
  "strconv"
@@ -46,23 +46,17 @@ type wapResBody struct {
  Message string `json:"message"`
 }
 
-type JobDataLanguage struct {
- Count int //数量
- Region string //地区
- Language string //语言
-}
-
 // 并行获取
 func GetJobs() {
- ch1, ch2 := make(chan JobDataLanguage), make(chan JobDataLanguage)
+ ch1, ch2 := make(chan models.JobDataLanguage), make(chan models.JobDataLanguage)
  go RequestByAjax3(ch1, "上海", "nodejs")
  go RequestByAjax3(ch2, "上海", "golang")
  i := 0 // 用于跳出for循环
  for {
  select {
  case c1 := <-ch1:
  if c1.Count != 0 {
- saveJob(c1)
+ models.SaveJob(c1)
  }
  i++
  if i == 2 {
@@ -71,7 +65,7 @@ func GetJobs() {
  }
  case c2 := <-ch2:
  if c2.Count != 0 {
- saveJob(c2)
+ models.SaveJob(c2)
  }
  i++
  if i == 2 {
@@ -85,7 +79,7 @@ ForEnd:
 }
 
 // 获取HTML页面中需要的数据
-func requestUrl(c chan JobDataLanguage, language, region string) {
+func requestUrl(c chan models.JobDataLanguage, language, region string) {
  //请求html数据
  res, err := http.Get(fmt.Sprintf("https://www.lagou.com/jobs/list_%s?px=default&city=%s#filterBox", language, url.QueryEscape(region)))
  if err != nil {
@@ -107,7 +101,7 @@ func requestUrl(c chan JobDataLanguage, language, region string) {
  text := doc.Find("#tab_pos>span").Text()
  logs.Info(language+"职位数：", text)
  count, _ := strconv.Atoi(text)
- c <- JobDataLanguage{
+ c <- models.JobDataLanguage{
  Count: count,
  Region: region,
  Language: language,
@@ -117,7 +111,7 @@ func requestUrl(c chan JobDataLanguage, language, region string) {
 /**
  * 通过ajax post获取数据
  */
-func RequestByAjax(c chan JobDataLanguage, language, region string) {
+func RequestByAjax(c chan models.JobDataLanguage, language, region string) {
  client := &http.Client{}
  // 使用wap端接口
  req, err := http.NewRequest(http.MethodGet,
@@ -162,7 +156,7 @@ func RequestByAjax(c chan JobDataLanguage, language, region string) {
  }
  countStr := resBody.Content.Data.Page.TotalCount
  count, _ := strconv.Atoi(countStr)
- c <- JobDataLanguage{
+ c <- models.JobDataLanguage{
  Count: count,
  Region: region,
  Language: language,
@@ -185,7 +179,7 @@ func GetCookies(url string) []string {
 }
 
 // 先访问页面获取cookie，再将cookie值放入请求header中
-func RequestByAjax3(c chan JobDataLanguage, region, language string) {
+func RequestByAjax3(c chan models.JobDataLanguage, region, language string) {
  cookies := GetCookies("https://m.lagou.com/search.html")
  //-------------------------------------
  req := httplib.Get(fmt.Sprintf("https://m.lagou.com/search.json?city=%s&positionName=%s&pageNo=1&pageSize=1", url.QueryEscape(region), language))
@@ -205,21 +199,10 @@ func RequestByAjax3(c chan JobDataLanguage, region, language string) {
  }
  countStr := resBody.Content.Data.Page.TotalCount
  count, _ := strconv.Atoi(countStr)
- j := JobDataLanguage{
+ j := models.JobDataLanguage{
  Count: count,
  Region: region,
  Language: language,
  }
  c <- j
 }
-
-// 保存数据
-func saveJob(c JobDataLanguage) {
- jobCount := &JobCount{
- JobTitle: c.Language,
- Amount: uint(c.Count),
- Region: c.Region,
- }
- jobCount.Save()
- logs.Info("保存job成功", c.Language, c.Count)
-}

util/helper.go

@@ -228,3 +228,14 @@ func Retry(attempts int, sleep time.Duration, f func(c int) error) (err error) {
  }
  return fmt.Errorf("after %d attempts, last error: %s", attempts, err)
 }
+
+func BuildInSQL[T any](arr []T) string {
+ qs := ""
+ for i, _ := range arr {
+ if i > 0 {
+ qs += ","
+ }
+ qs += "?"
+ }
+ return qs
+}

views/about.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html lang="zh-cn-Hans">
+<html lang="cmn-Hans">
 
 <head>
  <title>关于-月盾的网站</title>

views/admin/404.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html lang="zh-cn-Hans">
+<html lang="cmn-Hans">
 
 <head>
  <title>{{.content}}</title>

views/admin/article-add.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html lang="zh-cn-Hans">
+<html lang="cmn-Hans">
 
 <head>
  <title>新建文章</title>

views/admin/article-edit.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html lang="zh-cn-Hans">
+<html lang="cmn-Hans">
 
 <head>
  <title>修改文章</title>