Skip to content

Commit

Permalink
zerolog completelly useless for multiple go-routines ;( rs/zerolog#555,…
Browse files Browse the repository at this point in the history 
… try to remove all local loggers with context upload.go
  • Loading branch information
@Slach
Slach committed Jun 14, 2023
1 parent fd90f25 commit eea4f62
Show file tree
Hide file tree
Showing 21 changed files with 383 additions and 469 deletions.
2 changes: 0 additions & 2 deletions pkg/backup/backuper.go
View file
Expand Up @@ -7,7 +7,6 @@ import (
"github.com/Altinity/clickhouse-backup/pkg/config"
"github.com/Altinity/clickhouse-backup/pkg/resumable"
"github.com/Altinity/clickhouse-backup/pkg/storage"
"github.com/rs/zerolog/log"
"path"
)

Expand All @@ -26,7 +25,6 @@ type Backuper struct {
func NewBackuper(cfg *config.Config) *Backuper {
ch := &clickhouse.ClickHouse{
Config: &cfg.ClickHouse,
Logger: log.With().Str("logger", "clickhouse").Logger(),
}
return &Backuper{
cfg: cfg,
Expand Down
88 changes: 39 additions & 49 deletions pkg/backup/create.go
View file

Large diffs are not rendered by default.

17 changes: 7 additions & 10 deletions pkg/backup/delete.go
View file
Expand Up @@ -18,7 +18,6 @@ import (

// Clean - removed all data in shadow folder
func (b *Backuper) Clean(ctx context.Context) error {
logger := log.With().Str("logger", "Clean").Logger()
if err := b.ch.Connect(); err != nil {
return fmt.Errorf("can't connect to clickhouse: %v", err)
}
Expand All @@ -36,7 +35,7 @@ func (b *Backuper) Clean(ctx context.Context) error {
if err := b.cleanDir(shadowDir); err != nil {
return fmt.Errorf("can't clean '%s': %v", shadowDir, err)
}
logger.Info().Msg(shadowDir)
log.Info().Msg(shadowDir)
}
return nil
}
Expand Down Expand Up @@ -95,7 +94,6 @@ func (b *Backuper) RemoveOldBackupsLocal(ctx context.Context, keepLastBackup boo
}

func (b *Backuper) RemoveBackupLocal(ctx context.Context, backupName string, disks []clickhouse.Disk) error {
logger := log.With().Str("logger", "RemoveBackupLocal").Logger()
var err error
start := time.Now()
backupName = utils.CleanBackupNameRE.ReplaceAllString(backupName, "")
Expand All @@ -120,13 +118,13 @@ func (b *Backuper) RemoveBackupLocal(ctx context.Context, backupName string, dis
if disk.IsBackup {
backupPath = path.Join(disk.Path, backupName)
}
logger.Debug().Msgf("remove '%s'", backupPath)
log.Debug().Msgf("remove '%s'", backupPath)
err = os.RemoveAll(backupPath)
if err != nil {
return err
}
}
logger.Info().Str("operation", "delete").
log.Info().Str("operation", "delete").
Str("location", "local").
Str("backup", backupName).
Str("duration", utils.HumanizeDuration(time.Since(start))).
Expand All @@ -138,12 +136,11 @@ func (b *Backuper) RemoveBackupLocal(ctx context.Context, backupName string, dis
}

func (b *Backuper) RemoveBackupRemote(ctx context.Context, backupName string) error {
logger := log.With().Str("logger", "RemoveBackupRemote").Logger()
backupName = utils.CleanBackupNameRE.ReplaceAllString(backupName, "")
start := time.Now()
if b.cfg.General.RemoteStorage == "none" {
err := errors.New("aborted: RemoteStorage set to \"none\"")
logger.Error().Msg(err.Error())
log.Error().Msg(err.Error())
return err
}
if b.cfg.General.RemoteStorage == "custom" {
Expand All @@ -164,7 +161,7 @@ func (b *Backuper) RemoveBackupRemote(ctx context.Context, backupName string) er
}
defer func() {
if err := bd.Close(ctx); err != nil {
logger.Warn().Msgf("can't close BackupDestination error: %v", err)
log.Warn().Msgf("can't close BackupDestination error: %v", err)
}
}()

Expand All @@ -175,10 +172,10 @@ func (b *Backuper) RemoveBackupRemote(ctx context.Context, backupName string) er
for _, backup := range backupList {
if backup.BackupName == backupName {
if err := bd.RemoveBackup(ctx, backup); err != nil {
logger.Warn().Msgf("bd.RemoveBackup return error: %v", err)
log.Warn().Msgf("bd.RemoveBackup return error: %v", err)
return err
}
logger.Info().Fields(map[string]interface{}{
log.Info().Fields(map[string]interface{}{
"backup": backupName,
"location": "remote",
"operation": "delete",
Expand Down
147 changes: 71 additions & 76 deletions pkg/backup/download.go
View file

Large diffs are not rendered by default.

29 changes: 11 additions & 18 deletions pkg/backup/list.go
View file
Expand Up @@ -35,7 +35,6 @@ func (b *Backuper) List(what, format string) error {
return nil
}
func printBackupsRemote(w io.Writer, backupList []storage.Backup, format string) error {
logger := log.With().Str("logger", "printBackupsRemote").Logger()
switch format {
case "latest", "last", "l":
if len(backupList) < 1 {
Expand Down Expand Up @@ -73,7 +72,7 @@ func printBackupsRemote(w io.Writer, backupList []storage.Backup, format string)
size = "???"
}
if bytes, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", backup.BackupName, size, uploadDate, "remote", required, description); err != nil {
logger.Error().Msgf("fmt.Fprintf write %d bytes return error: %v", bytes, err)
log.Error().Msgf("fmt.Fprintf write %d bytes return error: %v", bytes, err)
}
}
default:
Expand All @@ -83,7 +82,6 @@ func printBackupsRemote(w io.Writer, backupList []storage.Backup, format string)
}

func printBackupsLocal(ctx context.Context, w io.Writer, backupList []LocalBackup, format string) error {
logger := log.With().Str("logger", "printBackupsLocal").Logger()
switch format {
case "latest", "last", "l":
if len(backupList) < 1 {
Expand Down Expand Up @@ -122,7 +120,7 @@ func printBackupsLocal(ctx context.Context, w io.Writer, backupList []LocalBacku
size = "???"
}
if bytes, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", backup.BackupName, size, creationDate, "local", required, description); err != nil {
logger.Error().Msgf("fmt.Fprintf write %d bytes return error: %v", bytes, err)
log.Error().Msgf("fmt.Fprintf write %d bytes return error: %v", bytes, err)
}
}
}
Expand All @@ -134,7 +132,6 @@ func printBackupsLocal(ctx context.Context, w io.Writer, backupList []LocalBacku

// PrintLocalBackups - print all backups stored locally
func (b *Backuper) PrintLocalBackups(ctx context.Context, format string) error {
logger := log.With().Str("logger", "PrintLocalBackups").Logger()
if !b.ch.IsOpen {
if err := b.ch.Connect(); err != nil {
return fmt.Errorf("can't connect to clickhouse: %v", err)
Expand All @@ -144,7 +141,7 @@ func (b *Backuper) PrintLocalBackups(ctx context.Context, format string) error {
w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', tabwriter.DiscardEmptyColumns)
defer func() {
if err := w.Flush(); err != nil {
logger.Error().Msgf("can't flush tabular writer error: %v", err)
log.Error().Msgf("can't flush tabular writer error: %v", err)
}
}()
backupList, _, err := b.GetLocalBackups(ctx, nil)
Expand All @@ -163,7 +160,6 @@ func (b *Backuper) GetLocalBackups(ctx context.Context, disks []clickhouse.Disk)
}
defer b.ch.Close()
}
logger := log.With().Str("logger", "GetLocalBackups").Logger()
if disks == nil {
disks, err = b.ch.GetDisks(ctx)
if err != nil {
Expand Down Expand Up @@ -244,7 +240,7 @@ func (b *Backuper) GetLocalBackups(ctx context.Context, disks []clickhouse.Disk)
})
}
if closeErr := d.Close(); closeErr != nil {
logger.Error().Msgf("can't close %s openError: %v", backupPath, closeErr)
log.Error().Msgf("can't close %s openError: %v", backupPath, closeErr)
}
}
}
Expand All @@ -262,18 +258,17 @@ func (b *Backuper) PrintAllBackups(ctx context.Context, format string) error {
}
defer b.ch.Close()
}
logger := log.With().Str("logger", "PrintAllBackups").Logger()
defer func() {
if err := w.Flush(); err != nil {
logger.Error().Msgf("can't flush tabular writer error: %v", err)
log.Error().Msgf("can't flush tabular writer error: %v", err)
}
}()
localBackups, _, err := b.GetLocalBackups(ctx, nil)
if err != nil && !os.IsNotExist(err) {
return err
}
if err = printBackupsLocal(ctx, w, localBackups, format); err != nil {
logger.Warn().Msgf("printBackupsLocal return error: %v", err)
log.Warn().Msgf("printBackupsLocal return error: %v", err)
}

if b.cfg.General.RemoteStorage != "none" {
Expand All @@ -282,7 +277,7 @@ func (b *Backuper) PrintAllBackups(ctx context.Context, format string) error {
return err
}
if err = printBackupsRemote(w, remoteBackups, format); err != nil {
logger.Warn().Msgf("printBackupsRemote return error: %v", err)
log.Warn().Msgf("printBackupsRemote return error: %v", err)
}
}
return nil
Expand All @@ -296,11 +291,10 @@ func (b *Backuper) PrintRemoteBackups(ctx context.Context, format string) error
}
defer b.ch.Close()
}
logger := log.With().Str("logger", "PrintRemoteBackups").Logger()
w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', tabwriter.DiscardEmptyColumns)
defer func() {
if err := w.Flush(); err != nil {
logger.Error().Msgf("can't flush tabular writer error: %v", err)
log.Error().Msgf("can't flush tabular writer error: %v", err)
}
}()
backupList, err := b.GetRemoteBackups(ctx, true)
Expand Down Expand Up @@ -392,7 +386,6 @@ func (b *Backuper) PrintTables(printAll bool, tablePattern string) error {
return fmt.Errorf("can't connect to clickhouse: %v", err)
}
defer b.ch.Close()
logger := log.With().Str("logger", "PrintTables").Logger()
allTables, err := b.GetTables(ctx, tablePattern)
if err != nil {
return err
Expand All @@ -412,16 +405,16 @@ func (b *Backuper) PrintTables(printAll bool, tablePattern string) error {
}
if table.Skip {
if bytes, err := fmt.Fprintf(w, "%s.%s\t%s\t%v\tskip\n", table.Database, table.Name, utils.FormatBytes(table.TotalBytes), strings.Join(tableDisks, ",")); err != nil {
logger.Error().Msgf("fmt.Fprintf write %d bytes return error: %v", bytes, err)
log.Error().Msgf("fmt.Fprintf write %d bytes return error: %v", bytes, err)
}
continue
}
if bytes, err := fmt.Fprintf(w, "%s.%s\t%s\t%v\t\n", table.Database, table.Name, utils.FormatBytes(table.TotalBytes), strings.Join(tableDisks, ",")); err != nil {
logger.Error().Msgf("fmt.Fprintf write %d bytes return error: %v", bytes, err)
log.Error().Msgf("fmt.Fprintf write %d bytes return error: %v", bytes, err)
}
}
if err := w.Flush(); err != nil {
logger.Error().Msgf("can't flush tabular writer error: %v", err)
log.Error().Msgf("can't flush tabular writer error: %v", err)
}
return nil
}

0 comments on commit eea4f62

Please sign in to comment.