generated from vshn/go-bootstrap
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
7 changed files
with
457 additions
and
47 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,184 @@ | ||
| package database | ||
|
|
||
| import ( | ||
| "context" | ||
| "database/sql" | ||
| "fmt" | ||
| "github.com/appuio/appuio-cloud-reporting/pkg/db" | ||
| "github.com/jmoiron/sqlx" | ||
| "github.com/vshn/cloudscale-metrics-collector/pkg/categoriesmodel" | ||
| "github.com/vshn/cloudscale-metrics-collector/pkg/datetimesmodel" | ||
| "github.com/vshn/cloudscale-metrics-collector/pkg/discountsmodel" | ||
| "github.com/vshn/cloudscale-metrics-collector/pkg/factsmodel" | ||
| "github.com/vshn/cloudscale-metrics-collector/pkg/productsmodel" | ||
| "github.com/vshn/cloudscale-metrics-collector/pkg/queriesmodel" | ||
| "github.com/vshn/cloudscale-metrics-collector/pkg/tenantsmodel" | ||
| "os" | ||
| "strings" | ||
| "time" | ||
|
|
||
| ctrl "sigs.k8s.io/controller-runtime" | ||
| ) | ||
|
|
||
| const ( | ||
| dbUrlEnvVariable = "ACR_DB_URL" | ||
| sourceQueryStorage = "object-storage-storage" | ||
| provider = "exoscale" | ||
| queryAndZone = sourceQueryStorage + ":" + provider | ||
| defaultUnit = "GBDay" | ||
| ) | ||
|
|
||
| var ( | ||
| productRow = db.Product{ | ||
| Source: queryAndZone, | ||
| Target: sql.NullString{String: "1402", Valid: true}, | ||
| Amount: 0.00066, | ||
| Unit: "GBDay", | ||
| During: db.InfiniteRange(), | ||
| } | ||
| discountRow = db.Discount{ | ||
| Source: sourceQueryStorage, | ||
| Discount: 0, | ||
| During: db.InfiniteRange(), | ||
| } | ||
| queryRow = db.Query{ | ||
| Name: queryAndZone, | ||
| Description: "Object Storage - Storage (exoscale.com)", | ||
| Query: "", | ||
| Unit: "GBDay", | ||
| During: db.InfiniteRange(), | ||
| } | ||
| ) | ||
|
|
||
| type AggregatedBucket struct { | ||
| Organization string | ||
| // Storage in bytes | ||
| StorageUsed int64 | ||
| } | ||
|
|
||
| type Database struct { | ||
| Url string | ||
| connection *sqlx.DB | ||
| } | ||
|
|
||
| // Configure loads environment variable which holds the database URL | ||
| func Configure() (database *Database, err error) { | ||
| databaseUrl := os.Getenv(dbUrlEnvVariable) | ||
| if databaseUrl == "" { | ||
| return nil, fmt.Errorf("cannot find environment variable %s", dbUrlEnvVariable) | ||
| } | ||
| return &Database{Url: databaseUrl}, nil | ||
| } | ||
|
|
||
| // OpenConnection opens a connection to the postgres database | ||
| func (d *Database) OpenConnection() error { | ||
| connection, err := db.Openx(d.Url) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot create a connection to the database: %w", err) | ||
| } | ||
| d.connection = connection | ||
| return nil | ||
| } | ||
|
|
||
| // CloseConnection closes the connection to the postgres database | ||
| func (d *Database) CloseConnection() error { | ||
| err := d.connection.Close() | ||
| if err != nil { | ||
| return fmt.Errorf("cannot close database connection: %w", err) | ||
| } | ||
| return err | ||
| } | ||
|
|
||
| // EnsureBucketUsage saves the aggregated buckets usage by namespace to the postgresql database | ||
| // To save the correct data to the database the function also matches a relevant product, discount (if any) and query. | ||
| // The storage usage is referred to a day before the application ran (yesterday) | ||
| func (d *Database) EnsureBucketUsage(ctx context.Context, namespace string, aggregatedBucket AggregatedBucket, billingDate time.Time) error { | ||
| log := ctrl.LoggerFrom(ctx) | ||
| log.Info("Saving buckets usage for namespace", "namespace", namespace, "storage used", aggregatedBucket.StorageUsed) | ||
|
|
||
| // start new transaction for actual work | ||
| tx, err := d.connection.BeginTxx(ctx, &sql.TxOptions{}) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot create database transaction for namespace %s: %w", namespace, err) | ||
| } | ||
|
|
||
| tenant, err := tenantsmodel.Ensure(ctx, tx, &db.Tenant{Source: aggregatedBucket.Organization}) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot ensure organization for namespace %s: %w", namespace, err) | ||
| } | ||
|
|
||
| category, err := categoriesmodel.Ensure(ctx, tx, &db.Category{Source: "exoscale" + ":" + namespace}) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot ensure category for namespace %s: %w", namespace, err) | ||
| } | ||
|
|
||
| dateTime := datetimesmodel.New(billingDate) | ||
| dateTime, err = datetimesmodel.Ensure(ctx, tx, dateTime) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot ensure date time for namespace %s: %w", namespace, err) | ||
| } | ||
|
|
||
| product, err := productsmodel.GetBestMatch(ctx, tx, getSourceString(namespace, aggregatedBucket.Organization), billingDate) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot get product best match for namespace %s: %w", namespace, err) | ||
| } | ||
|
|
||
| discount, err := discountsmodel.GetBestMatch(ctx, tx, getSourceString(namespace, aggregatedBucket.Organization), billingDate) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot get discount best match for namespace %s: %w", namespace, err) | ||
| } | ||
|
|
||
| query, err := queriesmodel.GetByName(ctx, tx, queryAndZone) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot get query by name for namespace %s: %w", namespace, err) | ||
| } | ||
|
|
||
| var quantity float64 | ||
| if query.Unit == defaultUnit { | ||
| quantity = float64(aggregatedBucket.StorageUsed) / 1000 / 1000 / 1000 | ||
| } else { | ||
| return fmt.Errorf("unknown query unit %s", query.Unit) | ||
| } | ||
| storageFact := factsmodel.New(dateTime, query, tenant, category, product, discount, quantity) | ||
| _, err = factsmodel.Ensure(ctx, tx, storageFact) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot save fact for namespace %s: %w", namespace, err) | ||
| } | ||
|
|
||
| err = tx.Commit() | ||
| if err != nil { | ||
| return fmt.Errorf("cannot commit transaction for buckets in namespace %s: %w", namespace, err) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // EnsureInitConfiguration ensures the minimum exoscale object storage configuration data is present in the database | ||
| // before saving buckets usage | ||
| func (d *Database) EnsureInitConfiguration(ctx context.Context) error { | ||
| tx, err := d.connection.BeginTxx(ctx, &sql.TxOptions{}) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot begin transaction for initial database configuration: %w", err) | ||
| } | ||
| defer tx.Rollback() | ||
| _, err = productsmodel.Ensure(ctx, tx, &productRow) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot ensure exoscale product model in the database: %w", err) | ||
| } | ||
| _, err = discountsmodel.Ensure(ctx, tx, &discountRow) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot ensure exoscale discount model in the database: %w", err) | ||
| } | ||
| _, err = queriesmodel.Ensure(ctx, tx, &queryRow) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot ensure exoscale query model in the database: %w", err) | ||
| } | ||
| err = tx.Commit() | ||
| if err != nil { | ||
| return fmt.Errorf("cannot commit transaction for initial database configuration: %w", err) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func getSourceString(namespace, organization string) string { | ||
| return strings.Join([]string{queryAndZone, organization, namespace}, ":") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.