Commit 5eb5ecac authored by Corentin Barreau's avatar Corentin Barreau
Browse files

Add: initial files

parent 3eaac075
package gocrawlhq
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
)
func (c *Client) Discovered(URLs []string, URLType string) (discoveredResponse *DiscoveredResponse, err error) {
expectedStatusCode := 201
discoveredResponse = new(DiscoveredResponse)
// build payload
payload := DiscoveredPayload{
Project: c.Project,
Type: URLType,
URLs: URLs,
}
jsonPayload, err := json.Marshal(payload)
if err != nil {
return discoveredResponse, err
}
// build request
req, err := http.NewRequest("POST", discoveredEndpoint, bytes.NewReader(jsonPayload))
if err != nil {
return discoveredResponse, err
}
req.Header.Add("X-Auth-Key", c.Key)
req.Header.Add("X-Auth-Secret", c.Secret)
req.Header.Add("User-Agent", "gocrawlhq/"+Version)
// execute request
resp, err := c.HTTPClient.Do(req)
if err != nil {
return discoveredResponse, err
}
defer resp.Body.Close()
// check response status code
if resp.StatusCode != expectedStatusCode {
return discoveredResponse, fmt.Errorf("non-%d status code: %d", expectedStatusCode, resp.StatusCode)
}
// decode response body
err = json.NewDecoder(resp.Body).Decode(discoveredResponse)
if err != nil {
return discoveredResponse, err
}
return discoveredResponse, err
}
package gocrawlhq
import (
"encoding/json"
"fmt"
"net/http"
)
func (c *Client) Feed(size int) (feedResponse *FeedResponse, err error) {
expectedStatusCode := 200
feedResponse = new(FeedResponse)
// build request
req, err := http.NewRequest("GET", feedEndpoint, nil)
if err != nil {
return feedResponse, err
}
req.Header.Add("X-Auth-Key", c.Key)
req.Header.Add("X-Auth-Secret", c.Secret)
req.Header.Add("User-Agent", "gocrawlhq/"+Version)
// execute request
resp, err := c.HTTPClient.Do(req)
if err != nil {
return feedResponse, err
}
defer resp.Body.Close()
// check response status code
if resp.StatusCode != expectedStatusCode {
return feedResponse, fmt.Errorf("non-%d status code: %d", expectedStatusCode, resp.StatusCode)
}
// decode response body
err = json.NewDecoder(resp.Body).Decode(feedResponse)
if err != nil {
return feedResponse, err
}
return feedResponse, err
}
package gocrawlhq
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
)
func (c *Client) Finished(URLs []URL, URLType string) (finishedResponse *FinishedResponse, err error) {
expectedStatusCode := 200
finishedResponse = new(FinishedResponse)
// build payload
payload := FinishedPayload{
Project: c.Project,
URLs: URLs,
}
jsonPayload, err := json.Marshal(payload)
if err != nil {
return finishedResponse, err
}
// build request
req, err := http.NewRequest("POST", finishedEndpoint, bytes.NewReader(jsonPayload))
if err != nil {
return finishedResponse, err
}
req.Header.Add("X-Auth-Key", c.Key)
req.Header.Add("X-Auth-Secret", c.Secret)
req.Header.Add("User-Agent", "gocrawlhq/"+Version)
// execute request
resp, err := c.HTTPClient.Do(req)
if err != nil {
return finishedResponse, err
}
defer resp.Body.Close()
// check response status code
if resp.StatusCode != expectedStatusCode {
return finishedResponse, fmt.Errorf("non-%d status code: %d", expectedStatusCode, resp.StatusCode)
}
// decode response body
err = json.NewDecoder(resp.Body).Decode(finishedResponse)
if err != nil {
return finishedResponse, err
}
return finishedResponse, err
}
module gocrawlhq
go 1.17
package gocrawlhq
import (
"net/http"
"path"
)
var (
discoveredEndpoint string
finishedEndpoint string
feedEndpoint string
Version = "1.0.0"
)
func Init(key, secret, project, HQAddress string) (c Client, err error) {
c.Key = key
c.Secret = secret
c.Project = project
c.HTTPClient = http.DefaultClient
c.HQAddress = HQAddress
discoveredEndpoint = path.Join(c.HQAddress + "discovered")
finishedEndpoint = path.Join(c.HQAddress + "finished")
feedEndpoint = path.Join(c.HQAddress + "feed")
return c, nil
}
package gocrawlhq
import "net/http"
type Client struct {
Key string
Secret string
Project string
HQAddress string
HTTPClient *http.Client
}
type URL struct {
ID int `json:"id"`
Value string `json:"value"`
}
type FeedResponse struct {
URLs []URL `json:"urls"`
}
type DiscoveredResponse struct {
Project string `json:"project"`
Type string `json:"type"`
}
type FinishedResponse struct {
Project string `json:"project"`
}
type DiscoveredPayload struct {
Project string `json:"project"`
Type string `json:"type"`
URLs []string `json:"urls"`
}
type FinishedPayload struct {
Project string `json:"project"`
URLs []URL `json:"urls"`
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment