type IJob interface {
    Id() string
}

type Job struct {
    id string
    // add your own fields here
}

func (j *Job) Id() string {
    return j.id
}

type IOutput interface {
    Record() *Record
    RecordKeys() []string
    RecordFlat() []any
    Job() IJob
}

type Record struct {
    J    *Job   `json:"-" csv:"-"`
}

func (r *Record) Record() *Record {
    return r
}

func (r *Record) RecordKeys() []string {
    ....
    keys := make([]string, numFields)
    ....
    return keys
}

func (r *Record) RecordFlat() []any {
    ....
    return slice
}

func (r *Record) Job() core.IJob {
    return r.J
}

type Spider struct {
  gos.ICoreSpider[*Record]
}

func New(ctx context.Context) (*Spider, <-chan error) {

  // use proxies
  // proxies := core.WithProxies("proxy_url1", "proxy_url2", ...)
  // core := gos.New[*Record]().WithClient(
  // 	gos.DefaultClient(proxies),
  // )

  core := gos.New[*Record]()

  // Add middlewares
  core.MiddlewareManager.Add(MIDDLEWARES...)
  // Add pipelines
  core.PipelineManager.Add(PIPELINES...)

  errCh := make(chan error)

  go func() {
 	errCh <- core.Start(ctx)
  }()

  return &Spider{
	core,
  }, errCh
}

// This is the entrypoint to the spider
func (s *Spider) StartRequest(ctx context.Context, job *Job) {
  // for each request we must call NewRequest() and never reuse it
  req := s.NewRequest()

  var headers http.Header

  /* GET is the request method, method chaining possible
  req.Url("<URL_HERE>").
  Meta("MY_KEY1", "MY_VALUE").
  Meta("MY_KEY2", true).
  Header(headers)
  */
    
  /* POST
  req.Url(<URL_HERE>)
  req.Method("POST")
  req.Body(<BODY_HERE>)
  */
    
  // call the next parse method
  s.Request(req, s.parse)
}

// can be called when spider exits
func (s *Spider) Close(ctx context.Context) {
}

func (s *Spider) parse(ctx context.Context, resp core.IResponseReader) {
  // response.Body()
  // response.StatusCode()
  // response.Header()
  // response.Bytes()
  // response.Meta("MY_KEY1")
	
  // yielding output pushes output to be processed by pipelines, also check output.go for the fields
  var data Record

  err := json.Unmarshal(resp.Bytes(), &data)
  if err != nil {
    log.Panicln(err)
  }

  // s.Yield(&data)
}

...
// Middlewares here
var MIDDLEWARES = []middlewaremanager.Middleware{
	middlewares.Retry(),
	middlewares.MultiCookieJar,
	middlewares.DupeFilter,
}

var export2CSV = pipelines.Export2CSV[*Record](pipelines.Export2CSVOpts{
	Filename: "itstimeitsnowornever.csv",
})

// Pipelines here
var PIPELINES = []pm.IPipeline[*Record]{
	export2CSV,
	// export2Json,
}
...

func main() {
  ctx, cancel := context.WithCancel(context.Background())

  var wg sync.WaitGroup
  wg.Add(1)

  spider, errCh := test1.New(ctx)
  go func() {
	defer wg.Done()

	err := <-errCh

	if err != nil && errors.Is(err, context.Canceled) {
		return
	}

	fmt.Printf("failed: %q", err)
  }()

  // start the scraper with a job, currently nil is passed but you can pass your job here
  spider.StartRequest(ctx, nil)

  OnTerminate(func() {
	fmt.Println("exit signal received: shutting down gracefully")
	cancel()
	wg.Wait()
  })

}

func New(ctx context.Context) (*Spider, <-chan error) {
    // default client options
    // proxies := gos.WithProxies("proxy_url1", "proxy_url2", ...)
     
    // core := gos.New[*Record]().WithClient(
    // 	  gos.DefaultClient(proxies),
    // )

    // we can also provide in our custom client
    // core := gos.New[*Record]().WithClient(myCustomHTTPClient)
}

// use export 2 csv pipeline
export2Csv := pipelines.Export2CSV[*scrapejsp.Record](pipelines.Export2CSVOpts{
	Filename: "itstimeitsnowornever.csv",
})

// use export 2 json pipeline
export2Json := pipelines.Export2JSON[*scrapejsp.Record](pipelines.Export2JSONOpts{
	Filename:  "itstimeitsnowornever.json",
	Immediate: true,
})

func myCustomPipelineGroup() *pm.Group[*Record] {
  // create a group
  pipelineGroup := pm.NewGroup[*Record]()

  pipelineGroup.Add(export2CSV)
  // pipelineGroup.Add(export2Json)
  return pipelineGroup
}

// Pipelines here
// Executed in the order they appear.
var PIPELINES = []pm.IPipeline[*Record]{
  export2CSV,
  // export2Json,
  // myCustomPipelineGroup(), // use group as if it were a single pipeline
}

var MIDDLEWARES = []middlewaremanager.Middleware{
	middlewares.Retry(),
	middlewares.MultiCookieJar,
	middlewares.DupeFilter,
}

func MultiCookieJar(next http.RoundTripper) http.RoundTripper {
	return core.MiddlewareFunc(func(req *http.Request) (*http.Response, error) {
		// you middleware custom code here
	})
}