From a9085d26f86adc8efeed69dae7b7e75172ca9bee Mon Sep 17 00:00:00 2001 From: ukewea <60734042+ukewea@users.noreply.github.com> Date: Fri, 26 May 2023 17:19:38 +0800 Subject: [PATCH] feat: add fetch all history data functions --- cmd/fetchall/fetchall.go | 118 ++++++++++++++++++ .../fetchrecent.go} | 4 +- pkg/cryptocompare/client.go | 75 +++++++++-- 3 files changed, 188 insertions(+), 9 deletions(-) create mode 100644 cmd/fetchall/fetchall.go rename cmd/{fetchdata.go => fetchrecent/fetchrecent.go} (95%) diff --git a/cmd/fetchall/fetchall.go b/cmd/fetchall/fetchall.go new file mode 100644 index 0000000..8090842 --- /dev/null +++ b/cmd/fetchall/fetchall.go @@ -0,0 +1,118 @@ +package main + +import ( + "fmt" + "os" + "time" + + "crypto_project/config" + "crypto_project/pkg/cryptocompare" + "crypto_project/pkg/db" + "crypto_project/pkg/models" + + "github.com/sirupsen/logrus" +) + +func main() { + log := logrus.New() + log.Out = os.Stdout + log.Level = logrus.DebugLevel + + conf, err := config.ReadConfig("config.toml") + if err != nil { + log.Fatal("Error reading config: ") + log.Panic(err) + } + + dsn := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%d sslmode=disable TimeZone=Asia/Taipei", + conf.Database.Host, conf.Database.Username, conf.Database.Password, conf.Database.DBName, conf.Database.Port) + + tradingSymbols := conf.Fetch.TradingSymbols + vsCurrency := conf.Fetch.VSCurrency + limitDaily := conf.Fetch.LimitDaily + limitHourly := conf.Fetch.LimitHourly + + log.Infof("Starting data fetch for symbols: %v", tradingSymbols) + + client := cryptocompare.NewClient(conf.Cryptocompare.APIKey, log) + + db, err := db.NewDB(dsn, log) + if err != nil { + log.Fatalf("Failed to connect to DB: %v", err) + panic(err) + } + + for _, symbol := range tradingSymbols { + log.Debugf("Fetching hourly data for symbol: %s", symbol) + + // Fetch and save hourly data + hourlyData, err := client.FetchAllHourlyOHLCVData(symbol, vsCurrency, limitHourly) + if err != nil { + log.Errorf("Failed to fetch hourly data for symbol: %s, error: %v", symbol, err) + panic(err) + } + + log.Debugf("Successfully fetched hourly data for symbol: %s", symbol) + + hourlyOHLCVData := make([]models.CryptoOHLCVHourly, len(hourlyData)) + for i, d := range hourlyData { + hourlyOHLCVData[i] = models.CryptoOHLCVHourly{ + CryptoOHLCV: models.CryptoOHLCV{ + TradingSymbol: symbol, + VsCurrency: vsCurrency, + Timestamp: time.Unix(d.Time, 0).UTC(), + Open: d.Open, + High: d.High, + Low: d.Low, + Close: d.Close, + VolumeFrom: d.VolumeFrom, + VolumeTo: d.VolumeTo, + }, + } + } + + if err := db.SaveHourlyOHLCData(hourlyOHLCVData); err != nil { + log.Errorf("Failed to save hourly data for symbol: %s, error: %v", symbol, err) + panic(err) + } + + log.Debugf("Successfully saved hourly data for symbol: %s", symbol) + + log.Debugf("Fetching daily data for symbol: %s", symbol) + + // Fetch and save daily data + dailyData, err := client.FetchAllDailyOHLCVData(symbol, vsCurrency, limitDaily) + if err != nil { + log.Errorf("Failed to fetch daily data for symbol: %s, error: %v", symbol, err) + panic(err) + } + + dailyOHLCVData := make([]models.CryptoOHLCVDaily, len(dailyData)) + for i, d := range dailyData { + dailyOHLCVData[i] = models.CryptoOHLCVDaily{ + CryptoOHLCV: models.CryptoOHLCV{ + TradingSymbol: symbol, + VsCurrency: vsCurrency, + Timestamp: time.Unix(d.Time, 0).UTC(), + Open: d.Open, + High: d.High, + Low: d.Low, + Close: d.Close, + VolumeFrom: d.VolumeFrom, + VolumeTo: d.VolumeTo, + }, + } + } + + log.Debugf("Successfully fetched daily data for symbol: %s", symbol) + + if err := db.SaveDailyOHLCData(dailyOHLCVData); err != nil { + log.Errorf("Failed to save daily data for symbol: %s, error: %v", symbol, err) + panic(err) + } + + log.Debugf("Successfully saved daily data for symbol: %s", symbol) + } + + log.Infof("Data fetch completed for symbols: %v", tradingSymbols) +} diff --git a/cmd/fetchdata.go b/cmd/fetchrecent/fetchrecent.go similarity index 95% rename from cmd/fetchdata.go rename to cmd/fetchrecent/fetchrecent.go index 7ed65c0..8cabc0f 100644 --- a/cmd/fetchdata.go +++ b/cmd/fetchrecent/fetchrecent.go @@ -46,7 +46,7 @@ func main() { log.Debugf("Fetching hourly data for symbol: %s", symbol) // Fetch and save hourly data - hourlyData, err := client.FetchHourlyOHLCData(symbol, vsCurrency, limitHourly) + hourlyData, err := client.FetchHourlyOHLCVData(symbol, vsCurrency, limitHourly) if err != nil { log.Errorf("Failed to fetch hourly data for symbol: %s, error: %v", symbol, err) panic(err) @@ -81,7 +81,7 @@ func main() { log.Debugf("Fetching daily data for symbol: %s", symbol) // Fetch and save daily data - dailyData, err := client.FetchDailyOHLCData(symbol, vsCurrency, limitDaily) + dailyData, err := client.FetchDailyOHLCVData(symbol, vsCurrency, limitDaily) if err != nil { log.Errorf("Failed to fetch daily data for symbol: %s, error: %v", symbol, err) panic(err) diff --git a/pkg/cryptocompare/client.go b/pkg/cryptocompare/client.go index f98274f..a27d139 100644 --- a/pkg/cryptocompare/client.go +++ b/pkg/cryptocompare/client.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "net/http" + "time" "github.com/shopspring/decimal" "github.com/sirupsen/logrus" @@ -14,6 +15,7 @@ const ( histohourEndpoint = "histohour" histodayEndpoint = "histoday" histominuteEndpoint = "histominute" + apiMaxLimit = 2000 ) type Client struct { @@ -40,6 +42,7 @@ type CryptoResponse struct { } `json:"Data"` } +// NewClient creates a new Client with given API key and logger func NewClient(apiKey string, logger *logrus.Logger) *Client { return &Client{ apiKey: apiKey, @@ -48,21 +51,79 @@ func NewClient(apiKey string, logger *logrus.Logger) *Client { } } -func (c *Client) FetchMinuteOHLCData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) { - return c.fetchOHLCData(tradingSymbol, vsCurrency, limit, histominuteEndpoint) +// FetchMinuteOHLCVData fetches minute-level OHLCV data up to given limit +func (c *Client) FetchMinuteOHLCVData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) { + c.logger.Trace("Fetching minute-level OHLCV data") + return c.fetchOHLCVData(tradingSymbol, vsCurrency, limit, histominuteEndpoint) } -func (c *Client) FetchHourlyOHLCData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) { - return c.fetchOHLCData(tradingSymbol, vsCurrency, limit, histohourEndpoint) +// FetchHourlyOHLCVData fetches hourly-level OHLCV data up to given limit +func (c *Client) FetchHourlyOHLCVData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) { + c.logger.Trace("Fetching hourly-level OHLCV data") + return c.fetchOHLCVData(tradingSymbol, vsCurrency, limit, histohourEndpoint) } -func (c *Client) FetchDailyOHLCData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) { - return c.fetchOHLCData(tradingSymbol, vsCurrency, limit, histodayEndpoint) +// FetchDailyOHLCVData fetches daily-level OHLCV data up to given limit +func (c *Client) FetchDailyOHLCVData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) { + c.logger.Trace("Fetching daily-level OHLCV data") + return c.fetchOHLCVData(tradingSymbol, vsCurrency, limit, histodayEndpoint) } -func (c *Client) fetchOHLCData(tradingSymbol, vsCurrency string, limit int, endpoint string) ([]OHLCVData, error) { +// FetchAllAllMinuteOHLCVData fetches all minute-level OHLCV data +func (c *Client) FetchAllAllMinuteOHLCVData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) { + c.logger.Trace("Fetching all minute-level OHLCV data") + return c.fetchAllOHLCVData(tradingSymbol, vsCurrency, histominuteEndpoint) +} + +// FetchAllHourlyOHLCVData fetches all available hourly-level OH// FetchAllHourlyOHLCVData fetches all available hourly-level OHLCV data from the CryptoCompare API. +func (c *Client) FetchAllHourlyOHLCVData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) { + c.logger.Trace("Initiating FetchAllHourlyOHLCVData request.") + return c.fetchAllOHLCVData(tradingSymbol, vsCurrency, histohourEndpoint) +} + +// FetchAllDailyOHLCVData fetches all available daily-level OHLCV data from the CryptoCompare API. +func (c *Client) FetchAllDailyOHLCVData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) { + c.logger.Trace("Initiating FetchAllDailyOHLCVData request.") + return c.fetchAllOHLCVData(tradingSymbol, vsCurrency, histodayEndpoint) +} + +// fetchAllOHLCVData fetches all available OHLCV data of a specific frequency from the CryptoCompare API. +func (c *Client) fetchAllOHLCVData(tradingSymbol, vsCurrency string, endpoint string) ([]OHLCVData, error) { + var allData []OHLCVData + var toTs int64 = time.Now().Unix() + + c.logger.Info("Starting fetchAllOHLCVData request.") + + for { + c.logger.Debug("Fetching more data in fetchAllOHLCVData.") + data, err := c.fetchOHLCVDataWithTs(tradingSymbol, vsCurrency, apiMaxLimit, endpoint, toTs) + if err != nil { + c.logger.Errorf("Error in fetchOHLCVDataWithTs: %v", err) + return nil, err + } + if len(data) == 0 { + break + } + + allData = append(allData, data...) + toTs = data[len(data)-1].Time + } + + c.logger.Info("Completed fetchAllOHLCVData request.") + return allData, nil +} + +func (c *Client) fetchOHLCVData(tradingSymbol, vsCurrency string, limit int, endpoint string) ([]OHLCVData, error) { url := fmt.Sprintf("%s/%s?fsym=%s&tsym=%s&limit=%d&api_key=%s", baseURL, endpoint, tradingSymbol, vsCurrency, limit, c.apiKey) + return c.getOHLCVDataFromApi(url) +} + +func (c *Client) fetchOHLCVDataWithTs(tradingSymbol, vsCurrency string, limit int, endpoint string, toTs int64) ([]OHLCVData, error) { + url := fmt.Sprintf("%s/%s?fsym=%s&tsym=%s&limit=%d&toTs=%d&api_key=%s", baseURL, endpoint, tradingSymbol, vsCurrency, limit, toTs, c.apiKey) + return c.getOHLCVDataFromApi(url) +} +func (c *Client) getOHLCVDataFromApi(url string) ([]OHLCVData, error) { c.logger.Debugf("Fetching data from URL: %s", url) resp, err := c.httpClient.Get(url)