Initial commit

.gitignore

@@ -0,0 +1,23 @@
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+
+config.toml

cmd/fetchdata.go

@@ -0,0 +1,118 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"crypto_project/config"
+	"crypto_project/pkg/cryptocompare"
+	"crypto_project/pkg/db"
+	"crypto_project/pkg/models"
+
+	"github.com/sirupsen/logrus"
+)
+
+func main() {
+	log := logrus.New()
+	log.Out = os.Stdout
+	log.Level = logrus.DebugLevel
+
+	conf, err := config.ReadConfig("config.toml")
+	if err != nil {
+		log.Fatal("Error reading config: ")
+		log.Panic(err)
+	}
+
+	dsn := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%d sslmode=disable TimeZone=Asia/Taipei",
+		conf.Database.Host, conf.Database.Username, conf.Database.Password, conf.Database.DBName, conf.Database.Port)
+
+	tradingSymbols := conf.Fetch.TradingSymbols
+	vsCurrency := conf.Fetch.VSCurrency
+	limitDaily := conf.Fetch.LimitDaily
+	limitHourly := conf.Fetch.LimitHourly
+
+	log.Infof("Starting data fetch for symbols: %v", tradingSymbols)
+
+	client := cryptocompare.NewClient(conf.Cryptocompare.APIKey, log)
+
+	db, err := db.NewDB(dsn, log)
+	if err != nil {
+		log.Fatalf("Failed to connect to DB: %v", err)
+		panic(err)
+	}
+
+	for _, symbol := range tradingSymbols {
+		log.Debugf("Fetching hourly data for symbol: %s", symbol)
+
+		// Fetch and save hourly data
+		hourlyData, err := client.FetchHourlyOHLCData(symbol, vsCurrency, limitHourly)
+		if err != nil {
+			log.Errorf("Failed to fetch hourly data for symbol: %s, error: %v", symbol, err)
+			panic(err)
+		}
+
+		log.Debugf("Successfully fetched hourly data for symbol: %s", symbol)
+
+		hourlyOHLCVData := make([]models.CryptoOHLCVHourly, len(hourlyData))
+		for i, d := range hourlyData {
+			hourlyOHLCVData[i] = models.CryptoOHLCVHourly{
+				CryptoOHLCV: models.CryptoOHLCV{
+					TradingSymbol: symbol,
+					VsCurrency:    vsCurrency,
+					Timestamp:     time.Unix(d.Time, 0).UTC(),
+					Open:          d.Open,
+					High:          d.High,
+					Low:           d.Low,
+					Close:         d.Close,
+					VolumeFrom:    d.VolumeFrom,
+					VolumeTo:      d.VolumeTo,
+				},
+			}
+		}
+
+		if err := db.SaveHourlyOHLCData(hourlyOHLCVData); err != nil {
+			log.Errorf("Failed to save hourly data for symbol: %s, error: %v", symbol, err)
+			panic(err)
+		}
+
+		log.Debugf("Successfully saved hourly data for symbol: %s", symbol)
+
+		log.Debugf("Fetching daily data for symbol: %s", symbol)
+
+		// Fetch and save daily data
+		dailyData, err := client.FetchDailyOHLCData(symbol, vsCurrency, limitDaily)
+		if err != nil {
+			log.Errorf("Failed to fetch daily data for symbol: %s, error: %v", symbol, err)
+			panic(err)
+		}
+
+		dailyOHLCVData := make([]models.CryptoOHLCVDaily, len(dailyData))
+		for i, d := range dailyData {
+			dailyOHLCVData[i] = models.CryptoOHLCVDaily{
+				CryptoOHLCV: models.CryptoOHLCV{
+					TradingSymbol: symbol,
+					VsCurrency:    vsCurrency,
+					Timestamp:     time.Unix(d.Time, 0).UTC(),
+					Open:          d.Open,
+					High:          d.High,
+					Low:           d.Low,
+					Close:         d.Close,
+					VolumeFrom:    d.VolumeFrom,
+					VolumeTo:      d.VolumeTo,
+				},
+			}
+		}
+
+		log.Debugf("Successfully fetched daily data for symbol: %s", symbol)
+
+		if err := db.SaveDailyOHLCData(dailyOHLCVData); err != nil {
+			log.Errorf("Failed to save daily data for symbol: %s, error: %v", symbol, err)
+			panic(err)
+		}
+
+		log.Debugf("Successfully saved daily data for symbol: %s", symbol)
+	}
+
+	log.Infof("Data fetch completed for symbols: %v", tradingSymbols)
+}

config.toml.EXAMPLE

@@ -0,0 +1,15 @@
+[database]
+host = "localhost"
+port = 5432
+username = "user"
+password = "pwd"
+db_name = "crypto_data"
+
+[cryptocompare]
+api_key = "key_from_cryptocompare"
+
+[fetch]
+trading_symbols = ["BTC", "DOGE", "ETH"]
+vs_currency = "USD"
+limit_daily = 7
+limit_hourly = 24

config/config.go

@@ -0,0 +1,30 @@
+package config
+
+import "github.com/BurntSushi/toml"
+
+type Config struct {
+	Database struct {
+		Host     string `toml:"host"`
+		Port     int    `toml:"port"`
+		Username string `toml:"username"`
+		Password string `toml:"password"`
+		DBName   string `toml:"dbname"`
+	} `toml:"database"`
+	Cryptocompare struct {
+		APIKey string `toml:"api_key"`
+	} `toml:"cryptocompare"`
+	Fetch struct {
+		TradingSymbols []string `toml:"trading_symbols"`
+		VSCurrency     string   `toml:"vs_currency"`
+		LimitDaily     int      `toml:"limit_daily"`
+		LimitHourly    int      `toml:"limit_hourly"`
+	} `toml:"fetch"`
+}
+
+func ReadConfig(filename string) (*Config, error) {
+	var conf Config
+	if _, err := toml.DecodeFile(filename, &conf); err != nil {
+		return nil, err
+	}
+	return &conf, nil
+}

go.mod

@@ -0,0 +1,22 @@
+module crypto_project
+
+go 1.18
+
+require (
+	github.com/BurntSushi/toml v1.2.1
+	github.com/shopspring/decimal v1.3.1
+	github.com/sirupsen/logrus v1.9.2
+	gorm.io/driver/postgres v1.5.2
+	gorm.io/gorm v1.25.1
+)
+
+require (
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/pgx/v5 v5.3.1 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
+	golang.org/x/crypto v0.9.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
+)

go.sum

@@ -0,0 +1,39 @@
+github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
+github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.3.1 h1:Fcr8QJ1ZeLi5zsPZqQeUZhNhxfkkKBOgJuYkJHoBOtU=
+github.com/jackc/pgx/v5 v5.3.1/go.mod h1:t3JDKnCBlYIc0ewLF0Q7B8MXmoIaBOZj/ic7iHozM/8=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
+github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y=
+github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
+golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gorm.io/driver/postgres v1.5.2 h1:ytTDxxEv+MplXOfFe3Lzm7SjG09fcdb3Z/c056DTBx0=
+gorm.io/driver/postgres v1.5.2/go.mod h1:fmpX0m2I1PKuR7mKZiEluwrP3hbs+ps7JIGMUBpCgl8=
+gorm.io/gorm v1.25.1 h1:nsSALe5Pr+cM3V1qwwQ7rOkw+6UeLrX5O4v3llhHa64=
+gorm.io/gorm v1.25.1/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=

pkg/cryptocompare/client.go

@@ -0,0 +1,89 @@
+package cryptocompare
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/shopspring/decimal"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	baseURL             = "https://min-api.cryptocompare.com/data/v2"
+	histohourEndpoint   = "histohour"
+	histodayEndpoint    = "histoday"
+	histominuteEndpoint = "histominute"
+)
+
+type Client struct {
+	apiKey     string
+	httpClient *http.Client
+	logger     *logrus.Logger
+}
+
+type OHLCVData struct {
+	Time       int64           `json:"time"`
+	Open       decimal.Decimal `json:"open"`
+	High       decimal.Decimal `json:"high"`
+	Low        decimal.Decimal `json:"low"`
+	Close      decimal.Decimal `json:"close"`
+	VolumeFrom decimal.Decimal `json:"volumefrom"`
+	VolumeTo   decimal.Decimal `json:"volumeto"`
+}
+
+type CryptoResponse struct {
+	Response string `json:"Response"`
+	Message  string `json:"Message"`
+	Data     struct {
+		Data []OHLCVData `json:"Data"`
+	} `json:"Data"`
+}
+
+func NewClient(apiKey string, logger *logrus.Logger) *Client {
+	return &Client{
+		apiKey:     apiKey,
+		httpClient: &http.Client{},
+		logger:     logrus.New(),
+	}
+}
+
+func (c *Client) FetchMinuteOHLCData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) {
+	return c.fetchOHLCData(tradingSymbol, vsCurrency, limit, histominuteEndpoint)
+}
+
+func (c *Client) FetchHourlyOHLCData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) {
+	return c.fetchOHLCData(tradingSymbol, vsCurrency, limit, histohourEndpoint)
+}
+
+func (c *Client) FetchDailyOHLCData(tradingSymbol, vsCurrency string, limit int) ([]OHLCVData, error) {
+	return c.fetchOHLCData(tradingSymbol, vsCurrency, limit, histodayEndpoint)
+}
+
+func (c *Client) fetchOHLCData(tradingSymbol, vsCurrency string, limit int, endpoint string) ([]OHLCVData, error) {
+	url := fmt.Sprintf("%s/%s?fsym=%s&tsym=%s&limit=%d&api_key=%s", baseURL, endpoint, tradingSymbol, vsCurrency, limit, c.apiKey)
+
+	c.logger.Debugf("Fetching data from URL: %s", url)
+
+	resp, err := c.httpClient.Get(url)
+	if err != nil {
+		c.logger.Errorf("Error making HTTP GET request: %v", err)
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var cr CryptoResponse
+	if err := json.NewDecoder(resp.Body).Decode(&cr); err != nil {
+		c.logger.Errorf("Error decoding HTTP response: %v", err)
+		return nil, err
+	}
+
+	if cr.Response == "Error" {
+		c.logger.Errorf("Error fetching data: %s", cr.Message)
+		return nil, fmt.Errorf("error fetching data: %s", cr.Message)
+	}
+
+	c.logger.Debugf("Successfully fetched data from URL: %s", url)
+
+	return cr.Data.Data, nil
+}