feat: add fetch all history data functions

cmd/fetchall/fetchall.go

@@ -0,0 +1,118 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"crypto_project/config"
+	"crypto_project/pkg/cryptocompare"
+	"crypto_project/pkg/db"
+	"crypto_project/pkg/models"
+
+	"github.com/sirupsen/logrus"
+)
+
+func main() {
+	log := logrus.New()
+	log.Out = os.Stdout
+	log.Level = logrus.DebugLevel
+
+	conf, err := config.ReadConfig("config.toml")
+	if err != nil {
+		log.Fatal("Error reading config: ")
+		log.Panic(err)
+	}
+
+	dsn := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%d sslmode=disable TimeZone=Asia/Taipei",
+		conf.Database.Host, conf.Database.Username, conf.Database.Password, conf.Database.DBName, conf.Database.Port)
+
+	tradingSymbols := conf.Fetch.TradingSymbols
+	vsCurrency := conf.Fetch.VSCurrency
+	limitDaily := conf.Fetch.LimitDaily
+	limitHourly := conf.Fetch.LimitHourly
+
+	log.Infof("Starting data fetch for symbols: %v", tradingSymbols)
+
+	client := cryptocompare.NewClient(conf.Cryptocompare.APIKey, log)
+
+	db, err := db.NewDB(dsn, log)
+	if err != nil {
+		log.Fatalf("Failed to connect to DB: %v", err)
+		panic(err)
+	}
+
+	for _, symbol := range tradingSymbols {
+		log.Debugf("Fetching hourly data for symbol: %s", symbol)
+
+		// Fetch and save hourly data
+		hourlyData, err := client.FetchAllHourlyOHLCVData(symbol, vsCurrency, limitHourly)
+		if err != nil {
+			log.Errorf("Failed to fetch hourly data for symbol: %s, error: %v", symbol, err)
+			panic(err)
+		}
+
+		log.Debugf("Successfully fetched hourly data for symbol: %s", symbol)
+
+		hourlyOHLCVData := make([]models.CryptoOHLCVHourly, len(hourlyData))
+		for i, d := range hourlyData {
+			hourlyOHLCVData[i] = models.CryptoOHLCVHourly{
+				CryptoOHLCV: models.CryptoOHLCV{
+					TradingSymbol: symbol,
+					VsCurrency:    vsCurrency,
+					Timestamp:     time.Unix(d.Time, 0).UTC(),
+					Open:          d.Open,
+					High:          d.High,
+					Low:           d.Low,
+					Close:         d.Close,
+					VolumeFrom:    d.VolumeFrom,
+					VolumeTo:      d.VolumeTo,
+				},
+			}
+		}
+
+		if err := db.SaveHourlyOHLCData(hourlyOHLCVData); err != nil {
+			log.Errorf("Failed to save hourly data for symbol: %s, error: %v", symbol, err)
+			panic(err)
+		}
+
+		log.Debugf("Successfully saved hourly data for symbol: %s", symbol)
+
+		log.Debugf("Fetching daily data for symbol: %s", symbol)
+
+		// Fetch and save daily data
+		dailyData, err := client.FetchAllDailyOHLCVData(symbol, vsCurrency, limitDaily)
+		if err != nil {
+			log.Errorf("Failed to fetch daily data for symbol: %s, error: %v", symbol, err)
+			panic(err)
+		}
+
+		dailyOHLCVData := make([]models.CryptoOHLCVDaily, len(dailyData))
+		for i, d := range dailyData {
+			dailyOHLCVData[i] = models.CryptoOHLCVDaily{
+				CryptoOHLCV: models.CryptoOHLCV{
+					TradingSymbol: symbol,
+					VsCurrency:    vsCurrency,
+					Timestamp:     time.Unix(d.Time, 0).UTC(),
+					Open:          d.Open,
+					High:          d.High,
+					Low:           d.Low,
+					Close:         d.Close,
+					VolumeFrom:    d.VolumeFrom,
+					VolumeTo:      d.VolumeTo,
+				},
+			}
+		}
+
+		log.Debugf("Successfully fetched daily data for symbol: %s", symbol)
+
+		if err := db.SaveDailyOHLCData(dailyOHLCVData); err != nil {
+			log.Errorf("Failed to save daily data for symbol: %s, error: %v", symbol, err)
+			panic(err)
+		}
+
+		log.Debugf("Successfully saved daily data for symbol: %s", symbol)
+	}
+
+	log.Infof("Data fetch completed for symbols: %v", tradingSymbols)
+}

cmd/fetchdata.go → cmd/fetchrecent/fetchrecent.go

@@ -46,7 +46,7 @@ func main() {
 		log.Debugf("Fetching hourly data for symbol: %s", symbol)
 
 		// Fetch and save hourly data
-		hourlyData, err := client.FetchHourlyOHLCData(symbol, vsCurrency, limitHourly)
+		hourlyData, err := client.FetchHourlyOHLCVData(symbol, vsCurrency, limitHourly)
 		if err != nil {
 			log.Errorf("Failed to fetch hourly data for symbol: %s, error: %v", symbol, err)
 			panic(err)
@@ -81,7 +81,7 @@ func main() {
 		log.Debugf("Fetching daily data for symbol: %s", symbol)
 
 		// Fetch and save daily data
-		dailyData, err := client.FetchDailyOHLCData(symbol, vsCurrency, limitDaily)
+		dailyData, err := client.FetchDailyOHLCVData(symbol, vsCurrency, limitDaily)
 		if err != nil {
 			log.Errorf("Failed to fetch daily data for symbol: %s, error: %v", symbol, err)
 			panic(err)

pkg/cryptocompare/client.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"time"
 
 	"github.com/shopspring/decimal"
 	"github.com/sirupsen/logrus"
@@ -14,6 +15,7 @@ const (
 	histohourEndpoint   = "histohour"
 	histodayEndpoint    = "histoday"
 	histominuteEndpoint = "histominute"
+	apiMaxLimit         = 2000
 )
 
 type Client struct {
@@ -40,6 +42,7 @@ type CryptoResponse struct {
 	} `json:"Data"`
 }
 
+// NewClient creates a new Client with given API key and logger
 func NewClient(apiKey string, logger *logrus.Logger) *Client {
 	return &Client{
 		apiKey:     apiKey,
@@ -48,21 +51,79 @@ func NewClient(apiKey string, logger *logrus.Logger) *Client {
 	}
 }
 
-func (c *Client) FetchMinuteOHLCData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) {
-	return c.fetchOHLCData(tradingSymbol, vsCurrency, limit, histominuteEndpoint)
+// FetchMinuteOHLCVData fetches minute-level OHLCV data up to given limit
+func (c *Client) FetchMinuteOHLCVData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) {
+	c.logger.Trace("Fetching minute-level OHLCV data")
+	return c.fetchOHLCVData(tradingSymbol, vsCurrency, limit, histominuteEndpoint)
 }
 
-func (c *Client) FetchHourlyOHLCData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) {
-	return c.fetchOHLCData(tradingSymbol, vsCurrency, limit, histohourEndpoint)
+// FetchHourlyOHLCVData fetches hourly-level OHLCV data up to given limit
+func (c *Client) FetchHourlyOHLCVData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) {
+	c.logger.Trace("Fetching hourly-level OHLCV data")
+	return c.fetchOHLCVData(tradingSymbol, vsCurrency, limit, histohourEndpoint)
 }
 
-func (c *Client) FetchDailyOHLCData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) {
-	return c.fetchOHLCData(tradingSymbol, vsCurrency, limit, histodayEndpoint)
+// FetchDailyOHLCVData fetches daily-level OHLCV data up to given limit
+func (c *Client) FetchDailyOHLCVData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) {
+	c.logger.Trace("Fetching daily-level OHLCV data")
+	return c.fetchOHLCVData(tradingSymbol, vsCurrency, limit, histodayEndpoint)
 }
 
-func (c *Client) fetchOHLCData(tradingSymbol, vsCurrency string, limit int, endpoint string) ([]OHLCVData, error) {
+// FetchAllAllMinuteOHLCVData fetches all minute-level OHLCV data
+func (c *Client) FetchAllAllMinuteOHLCVData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) {
+	c.logger.Trace("Fetching all minute-level OHLCV data")
+	return c.fetchAllOHLCVData(tradingSymbol, vsCurrency, histominuteEndpoint)
+}
+
+// FetchAllHourlyOHLCVData fetches all available hourly-level OH// FetchAllHourlyOHLCVData fetches all available hourly-level OHLCV data from the CryptoCompare API.
+func (c *Client) FetchAllHourlyOHLCVData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) {
+	c.logger.Trace("Initiating FetchAllHourlyOHLCVData request.")
+	return c.fetchAllOHLCVData(tradingSymbol, vsCurrency, histohourEndpoint)
+}
+
+// FetchAllDailyOHLCVData fetches all available daily-level OHLCV data from the CryptoCompare API.
+func (c *Client) FetchAllDailyOHLCVData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) {
+	c.logger.Trace("Initiating FetchAllDailyOHLCVData request.")
+	return c.fetchAllOHLCVData(tradingSymbol, vsCurrency, histodayEndpoint)
+}
+
+// fetchAllOHLCVData fetches all available OHLCV data of a specific frequency from the CryptoCompare API.
+func (c *Client) fetchAllOHLCVData(tradingSymbol, vsCurrency string, endpoint string) ([]OHLCVData, error) {
+	var allData []OHLCVData
+	var toTs int64 = time.Now().Unix()
+
+	c.logger.Info("Starting fetchAllOHLCVData request.")
+
+	for {
+		c.logger.Debug("Fetching more data in fetchAllOHLCVData.")
+		data, err := c.fetchOHLCVDataWithTs(tradingSymbol, vsCurrency, apiMaxLimit, endpoint, toTs)
+		if err != nil {
+			c.logger.Errorf("Error in fetchOHLCVDataWithTs: %v", err)
+			return nil, err
+		}
+		if len(data) == 0 {
+			break
+		}
+
+		allData = append(allData, data...)
+		toTs = data[len(data)-1].Time
+	}
+
+	c.logger.Info("Completed fetchAllOHLCVData request.")
+	return allData, nil
+}
+
+func (c *Client) fetchOHLCVData(tradingSymbol, vsCurrency string, limit int, endpoint string) ([]OHLCVData, error) {
 	url := fmt.Sprintf("%s/%s?fsym=%s&tsym=%s&limit=%d&api_key=%s", baseURL, endpoint, tradingSymbol, vsCurrency, limit, c.apiKey)
+	return c.getOHLCVDataFromApi(url)
+}
+
+func (c *Client) fetchOHLCVDataWithTs(tradingSymbol, vsCurrency string, limit int, endpoint string, toTs int64) ([]OHLCVData, error) {
+	url := fmt.Sprintf("%s/%s?fsym=%s&tsym=%s&limit=%d&toTs=%d&api_key=%s", baseURL, endpoint, tradingSymbol, vsCurrency, limit, toTs, c.apiKey)
+	return c.getOHLCVDataFromApi(url)
+}
 
+func (c *Client) getOHLCVDataFromApi(url string) ([]OHLCVData, error) {
 	c.logger.Debugf("Fetching data from URL: %s", url)
 
 	resp, err := c.httpClient.Get(url)