/
push-build.sh
executable file
·1663 lines (1453 loc) · 55.7 KB
/
push-build.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env bash
# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Set PROGram name
PROG=${0##*/}
########################################################################
#+
#+ NAME
#+ $PROG - Push Kubernetes Release Artifacts up to GCS
#+
#+ SYNOPSIS
#+ $PROG [--nomock] [--federation] [--noupdatelatest] [--ci]
#+ [--bucket=<GS bucket>]
#+ [--private-bucket]
#+ $PROG [--helpshort|--usage|-?]
#+ $PROG [--help|-man]
#+
#+ DESCRIPTION
#+ Replaces kubernetes/build/push-*-build.sh.
#+ Used for pushing developer builds and Jenkins' continuous builds.
#+
#+ Developer pushes simply run as they do pushing to devel/ on GCS.
#+ In --ci mode, $PROG runs in mock mode by default. Use --nomock to do
#+ a real push.
#+
#+ Federation values are just passed through as exported global vars still
#+ due to the fact that we're still leveraging the existing federation
#+ interface in kubernetes proper.
#+
#+ OPTIONS
#+ [--nomock] - Enables a real push (--ci only)
#+ [--federation] - Enable FEDERATION push
#+ [--ci] - Used when called from Jenkins (for ci
#+ runs)
#+ [--fast] - Specifies a fast build (linux amd64 only)
#+ [--extra-publish-file=] - [DEPRECATED - use --extra-version-markers
#+ instead] Used when need to upload
#+ additional version file to GCS. The path
#+ is relative and is append to a GCS path.
#+ (--ci only)
#+ [--extra-version-markers=] - Used when need to upload additional
#+ version markers to GCS. The path is
#+ relative and is append to a GCS path.
#+ (--ci only)
#+ [--bucket=] - Specify an alternate bucket for pushes
#+ [--release-type=] - Override auto-detected release type
#+ (normally devel or ci)
#+ [--release-kind=] - Kind of release to push to GCS. Supported
#+ values are kubernetes(default) or
#+ federation.
#+ [--gcs-suffix=] - Specify a suffix to append to the upload
#+ destination on GCS.
#+ [--docker-registry=] - If set, push docker images to specified
#+ registry/project
#+ [--version-suffix=] - Append suffix to version name if set.
#+ [--noupdatelatest] - Do not update the latest file
#+ [--private-bucket] - Do not mark published bits on GCS as
#+ publicly readable.
#+ [--allow-dup] - Do not exit error if the build already
#+ exists on the gcs path.
#+ [--help | -man] - display man page for this script
#+ [--usage | -?] - display in-line usage
#+
#+ EXAMPLES
#+ $PROG - Do a developer push
#+ $PROG --nomock --federation --ci
#+ - Do a (non-mocked) CI push with federation
#+ $PROG --bucket=kubernetes-release-$USER
#+ - Do a developer push to
#+ kubernetes-release-$USER
#+
#+ BUGS/TODO
#+ * Remove the script if nobody is using it any more.
#+
########################################################################
# If NO ARGUMENTS should return *usage*, uncomment the following line:
#usage=${1:-yes}
if [[ $(uname) == "Darwin" ]]; then
READLINK_CMD=greadlink
else
READLINK_CMD=readlink
fi
##############################################################################
# common.sh
##############################################################################
# Provide a default $PROG (for use in most functions that use a $PROG: prefix
: ${PROG:="common"}
export PROG
##############################################################################
# Common library of useful functions and GLOBALS.
##############################################################################
set -o errtrace
declare -A some_var || (echo "Bash version >= 4.0 required" && exit 1)
if [[ $(uname) == "Darwin" ]]; then
# Support for OSX.
READLINK_CMD=greadlink
LC_ALL=C # To make BSD sed work with double-quoted strings.
else
READLINK_CMD=readlink
fi
##############################################################################
# COMMON CONSTANTS
##############################################################################
TOOL_LIB_PATH=${TOOL_LIB_PATH:-$(dirname $($READLINK_CMD -ne $BASH_SOURCE))}
TOOL_ROOT=${TOOL_ROOT:-$($READLINK_CMD -ne $TOOL_LIB_PATH/..)}
PATH=$TOOL_ROOT:$PATH
# Provide a default EDITOR for those that don't have this set
: ${EDITOR:="vi"}
export PATH TOOL_ROOT TOOL_LIB_PATH EDITOR
# Pretty curses stuff for terminals
if [[ -t 1 ]]; then
# Set some video text attributes for use in error/warning msgs.
declare -A TPUT=([BOLD]=$(tput bold 2>/dev/null))
TPUT+=(
[REVERSE]=$(tput rev 2>/dev/null)
[UNDERLINE]=$(tput smul 2>/dev/null)
[BLINK]=$(tput blink 2>/dev/null)
[GREEN]=${TPUT[BOLD]}$(tput setaf 2 2>/dev/null)
[RED]=${TPUT[BOLD]}$(tput setaf 1 2>/dev/null)
[YELLOW]=${TPUT[BOLD]}$(tput setaf 3 2>/dev/null)
[OFF]=$(tput sgr0 2>/dev/null)
[COLS]=$(tput cols 2>/dev/null)
)
# HR
HR="$(for ((i=1;i<=${TPUT[COLS]};i++)); do echo -en '\u2500'; done)"
# Save original TTY State
TTY_SAVED_STATE="$(stty -g)"
else
HR="$(for ((i=1;i<=80;i++)); do echo -en '='; done)"
fi
# Set some usable highlighted keywords for functions like logrun -s
YES="${TPUT[GREEN]}YES${TPUT[OFF]}"
OK="${TPUT[GREEN]}OK${TPUT[OFF]}"
DONE="${TPUT[GREEN]}DONE${TPUT[OFF]}"
PASSED="${TPUT[GREEN]}PASSED${TPUT[OFF]}"
FAILED="${TPUT[RED]}FAILED${TPUT[OFF]}"
FATAL="${TPUT[RED]}FATAL${TPUT[OFF]}"
NO="${TPUT[RED]}NO${TPUT[OFF]}"
WARNING="${TPUT[YELLOW]}WARNING${TPUT[OFF]}"
ATTENTION="${TPUT[YELLOW]}ATTENTION${TPUT[OFF]}"
MOCK="${TPUT[YELLOW]}MOCK${TPUT[OFF]}"
FOUND="${TPUT[GREEN]}FOUND${TPUT[OFF]}"
NOTFOUND="${TPUT[YELLOW]}NOT FOUND${TPUT[OFF]}"
# Ensure USER is set
USER=${USER:-$LOGNAME}
# Set a PID for use throughout.
export PID=$$
# Save original cmd-line.
ORIG_CMDLINE="$*"
# Global arrays and dictionaries for use with common::stepheader()
# and common::stepindex()
declare -A PROGSTEP
declare -a PROGSTEPS
declare -A PROGSTEPS_INDEX
###############################################################################
# Define logecho() function to display to both log and stdout.
# As this is widely used and to reduce clutter, we forgo the common:: prefix
# Options can be -n or -p or -np/-pn.
# @optparam -p Add $PROG: prefix to stdout
# @optparam -r Exclude log prefix (used to output status' like $OK $FAILED)
# @optparam -n no newline (just like echo -n)
# @param a string to echo to stdout
logecho () {
local log_prefix="$PROG::${FUNCNAME[1]:-"main"}(): "
local prefix
# Dynamically set fmtlen
local fmtlen=$((${TPUT[COLS]:-"90"}))
local n
local raw=0
#local -a sed_pat=()
while [[ "$#" -gt 0 ]]; do
case "$1" in
-r) raw=1; shift ;;
-n) n="-n"; shift ;;
-p) prefix="$PROG: "; ((fmtlen+=${#prefix})); shift ;;
*) break ;;
esac
done
if ((raw)) || [[ -z "$*" ]]; then
# Clean log_prefix for blank lines
log_prefix=""
#else
# Increase fmtlen to account for control characters
#((fmtlen+=$(echo "$*" grep -o '[[:cntrl:]]' |wc -l)))
#sed_pat=(-e '2,${s}/^/ ... /g')
fi
# Allow widespread use of logecho without having to
# determine if $LOGFILE exists first.
[[ -f $LOGFILE ]] || LOGFILE="/dev/null"
(
# If -n is set, do not provide autoformatting or you lose the -n effect
# Use of -n should only be used on short status lines anyway.
if ((raw)) || [[ $n == "-n" ]]; then
echo -e $n "$log_prefix$*"
else
# Add FUNCNAME to line prefix, but strip it from visible output
# Useful for viewing log detail
echo -e "$*" | fmt -$fmtlen | sed -e "1s,^,$log_prefix,g" "${sed_pat[@]}"
fi
) | tee -a "$LOGFILE" |sed "s,^$log_prefix,$prefix,g"
}
###############################################################################
# logrun() function to run commands to both log and stdout.
# As this is widely used and to reduce clutter, we forgo the common:: prefix
#
# The calling function is added to the line prefix.
# NOTE: All optparam's for logrun() (obviously) must precede the command string
# @optparam -v Run verbosely
# @optparam -s Provide a $OK or $FAILED status from running command
# @optparam -m MOCK command by printing out command line rather than running it.
# @optparam -r Retry attempts. Integer arg follows -r (Ex. -r 2)
# Typically used together with -v to show retry attempts.
# @param a command string
# GLOBALS used in this function:
# * LOGFILE (Set by common::logfileinit()), if set, gets full command output
# * FLAGS_verbose (Set by caller - defaults to false), if true, full output to stdout
logrun () {
local mock=0
local status=0
local arg
local retries=0
local try
local retry_string
local scope="::${FUNCNAME[1]:-main}()"
local ret
local verbose=0
while [[ "$#" -gt 0 ]]; do
case "$1" in
-v) verbose=1; shift ;;
-s) status=1; shift ;;
-m) mock=1; shift ;;
-r) retries=$2; shift 2;;
*) break ;;
esac
done
for ((try=0; try<=$retries; try++)); do
if [[ $try -gt 0 ]]; then
if ((verbose)) || ((FLAGS_verbose)); then
# if global FLAGS_verbose, be very verbose
logecho "Retry #$try..."
elif ((status)); then
# if we're reporting a status (-v), then just ...
logecho -n "."
fi
# Add some minimal wait between retries assuming we're retrying due to
# something resolvable by waiting 'just a bit'
sleep 2
fi
# if no args, take stdin
if (($#==0)); then
if ((verbose)) || ((FLAGS_verbose)); then
tee -a $LOGFILE
else
tee -a $LOGFILE &>/dev/null
fi
ret=$?
elif [[ -f "$LOGFILE" ]]; then
printf "\n$PROG$scope: %s\n" "$*" >> $LOGFILE
if ((mock)); then
logecho "($MOCK)"
logecho "(CMD): $@"
return 0
fi
# Special case "cd" which cannot be run through a pipe (subshell)
if (! ((FLAGS_verbose)) && ! ((verbose)) ) || [[ "$1" == "cd" ]]; then
"${@:-:}" >> $LOGFILE 2>&1
else
printf "\n$PROG$scope: %s\n" "$*"
"${@:-:}" 2>&1 | tee -a $LOGFILE
fi
ret=${PIPESTATUS[0]}
else
if ((mock)); then
logecho "($MOCK)"
logecho "(CMD): $@"
return 0
fi
if ((verbose)) || ((FLAGS_verbose)); then
printf "\n$PROG$scope: %s\n" "$*"
"${@:-:}"
else
"${@:-:}" &>/dev/null
fi
ret=${PIPESTATUS[0]}
fi
[[ "$ret" = 0 ]] && break
done
[[ -n "$retries" && $try > 0 ]] && retry_string=" (retry #$try)"
if ((status)); then
[[ "$ret" = 0 ]] && logecho -r "$OK$retry_string"
[[ "$ret" != 0 ]] && logecho -r "$FAILED"
fi
return $ret
}
###############################################################################
# common::timestamp() Capture block timings and display them
# The calling function is added to the line prefix.
# NOTE: All optparam's for logrun() (obviously) must precede the command string
# @param begin|end|done
# @optparam section defaults to main, but can be specified to time sub sections
common::timestamp () {
local action=$1
local section=${2:-main}
# convert illegal characters to (legal) underscore
section_var=${section//[-\.:\/]/_}
local start_var="${section_var}start_seconds"
local end_var="${section_var}end_seconds"
local prefix
local elapsed
local d
local h
local m
local s
local prettyd
local prettyh
local prettym
local prettys
local pretty
# Only prefix for "main"
if [[ $section == "main" ]]; then
prefix="$PROG: "
fi
case $action in
begin)
# Get time(date) for display and calc.
eval $start_var=$(date '+%s')
if [[ $section == "main" ]]; then
# Print BEGIN message for $PROG.
echo "${prefix}BEGIN $section on ${HOSTNAME%%.*} $(date)"
else
echo "$(date +[%Y-%b-%d\ %R:%S\ %Z]) $section"
fi
if [[ $section == "main" ]]; then
echo
fi
;;
end|done)
# Check for "START" values before calcing.
if [[ -z ${!start_var} ]]; then
#display_time="EE:EE:EE - 'end' run without 'begin' in this scope or sourced script using common::timestamp"
return 0
fi
# Get time(date) for display and calc.
eval $end_var=$(date '+%s')
elapsed=$(( ${!end_var} - ${!start_var} ))
d=$(( elapsed / 86400 ))
h=$(( (elapsed % 86400) / 3600 ))
m=$(( (elapsed % 3600) / 60 ))
s=$(( elapsed % 60 ))
(($d>0)) && local prettyd="${d}d"
(($h>0)) && local prettyh="${h}h"
(($m>0)) && local prettym="${m}m"
prettys="${s}s"
pretty="$prettyd$prettyh$prettym$prettys"
if [[ $section == "main" ]]; then
echo
echo "${prefix}DONE $section on ${HOSTNAME%%.*} $(date) in $pretty"
else
echo "$(date +[%Y-%b-%d\ %R:%S\ %Z]) $section in $pretty"
fi
;;
esac
}
# Write our own trap to capture signal
common::trap () {
local func="$1"
shift
local sig
for sig; do
trap "$func $sig" "$sig"
done
}
common::trapclean () {
local sig=$1
local frame=0
# If user ^C's at read then tty is hosed, so make it sane again.
[[ -n "$TTY_SAVED_STATE" ]] && stty "$TTY_SAVED_STATE"
logecho;logecho
logecho "Signal $sig caught!"
logecho
logecho "Traceback (line function script):"
while caller $frame; do
((frame++))
done
common::exit 2 "Exiting..."
}
#############################################################################
# Clean exit with an ending timestamp
# @param Exit code
common::cleanexit () {
# Display end common::timestamp when an existing common::timestamp begin
# was run.
[[ -n ${mainstart_seconds} ]] && common::timestamp end
exit ${1:-0}
}
#############################################################################
# common::cleanexit() entry point with some formatting and message printing
# @param Exit code
# @param message
common::exit () {
local etype=${1:-0}
shift
[[ -n "$1" ]] && (logecho;logecho "$@";logecho)
common::cleanexit $etype
}
#############################################################################
# Simple yes/no prompt
#
# @optparam default -n(default)/-y/-e (default to n, y or make (e)xplicit)
# @param message
common::askyorn () {
local yorn
local def=n
local msg="y/N"
case $1 in
-y) # yes default
def="y" msg="Y/n"
shift
;;
-e) # Explicit
def="" msg="y/n"
shift
;;
-n) shift
;;
esac
while [[ $yorn != [yYnN] ]]; do
logecho -n "$*? ($msg): "
read yorn
: ${yorn:=$def}
done
# Final test to set return code
[[ $yorn == [yY] ]]
}
###############################################################################
# Print PROGSTEPs as bolded headers within scripts.
# PROGSTEP is a globally defined dictionary (associative array) that can
# take a function name or integer as its key
# The function indexes the dictionary in the order the items are added (by
# calling the function) so that progress can be shown during script execution
# (1/4, 2/4...4/4)
# If a PROGSTEP dictionary is empty, common::stepheader() will just show the
# text passed in.
common::stepheader () {
# If called with no args, assume the key is the caller's function name
local key="${1:-${FUNCNAME[1]}}"
local append="$2"
local msg="${PROGSTEP[$key]:-$key}"
local index=
# Only display an index if the $key is part of one
[[ -n "${PROGSTEPS_INDEX[$key]:-}" ]] \
&& index="(${PROGSTEPS_INDEX[$key]}/${#PROGSTEPS_INDEX[@]})"
logecho
logecho -r "$HR"
logecho "$msg" "$append" $index
logecho -r "$HR"
logecho
}
# Save a specified number of backups to a file
common::rotatelog () {
local file=$1
local num=$2
local tmpfile=$TMPDIR/rotatelog.$PID
local counter=$num
# Quiet exit
[[ ! -f "$file" ]] && return
cp -p $file $tmpfile
while ((counter>=0)); do
if ((counter==num)); then
rm -f $file.$counter
elif ((counter==0)); then
if [[ -f "$file" ]]; then
next=$((counter+1))
mv $file $file.$next
fi
else
next=$((counter+1))
[[ -f $file.$counter ]] && mv $file.$counter $file.$next
fi
((counter==0)) && break
((counter--))
done
mv $tmpfile $file
}
# --norotate assumes you're passing in a unique LOGFILE.
# $2 then indicates the number of unique filenames prefixed up to the last
# dot extension that will be saved. The rest of those files will be deleted
# For example, common::logfileinit --norotate foo.log.234 100
# common::logfileinit maintains up to 100 foo.log.* files. Anything else named
# foo.log.* > 100 are removed.
common::logfileinit () {
local nr=false
if [[ "$1" == "--norotate" ]]; then
local nr=true
shift
fi
LOGFILE=${1:-$PWD/$PROG.log}
local num=$2
# Ensure LOG directory exists
mkdir -p $(dirname $LOGFILE 2>/dev/null)
# Initialize Logfile.
if ! $nr; then
common::rotatelog "$LOGFILE" ${num:-3}
fi
# Truncate the logfile.
> "$LOGFILE"
echo "CMD: $PROG $ORIG_CMDLINE" >> "$LOGFILE"
# with --norotate, remove the list of files that start with $PROG.log
if $nr; then
ls -1tr ${LOGFILE%.*}.* |head --lines=-$num |xargs rm -f
fi
}
# An alternative that has a dependency on external program - pandoc
# store markdown man pages in companion files. Allow prog -man to still read
# those and display a man page using:
# pandoc -s -f markdown -t man prog.md |man -l -
common::manpage () {
[[ "$usage" == "yes" ]] && set -- -usage
[[ "$man" == "yes" ]] && set -- -man
[[ "$comments" == "yes" ]] && set -- -comments
case $1 in
-*usage|"-?")
sed -n '/#+ SYNOPSIS/,/^#+ DESCRIPTION/p' $0 |sed '/^#+ DESCRIPTION/d' |\
envsubst | sed -e 's,^#+ ,,g' -e 's,^#+$,,g'
exit 1
;;
-*man|-h|-*help)
grep "^#+" "$0" |\
sed -e 's,^#+ ,,g' -e 's,^#+$,,g' |envsubst |${PAGER:-"less"}
exit 1
;;
esac
}
###############################################################################
# General command-line parser converting -*arg="value" to $FLAGS_arg="value"
# Set -name/--name booleans to FLAGS_name=1
# As a convenience, flags can contain dashes or underscores, but dashes are
# converted to underscores in the final FLAGS_name to conform to variable
# naming standards.
# Sets global array POSITIONAL_ARGV holding all non-dash command-line arguments
common::namevalue () {
local arg
local name
local value
local -A arg_aliases=([v]="verbose" [n]="dryrun")
for arg in "$@"; do
case $arg in
-*[[:alnum:]]*) # Strip off any leading - or --
arg=$(printf "%s\n" $arg |sed 's/^-\{1,2\}//')
# Handle global aliases
arg=${arg_aliases[$arg]:-"$arg"}
if [[ $arg =~ =(.*) ]]; then
name=${arg%%=*}
value=${arg#*=}
# change -'s to _ in name for legal vars in bash
eval export FLAGS_${name//-/_}=\""$value"\"
else
# bool=1
# change -'s to _ in name for legal vars in bash
eval export FLAGS_${arg//-/_}=1
fi
;;
*) POSITIONAL_ARGV+=($arg)
;;
esac
done
}
###############################################################################
# Simple argc validation with a usage return
# @param num - number of POSITIONAL_ARGV that should be on the command-line
# return 1 if any number other than num
common::argc_validate () {
local args=$1
# Validate number of args
if ((${#POSITIONAL_ARGV[@]}>args)); then
logecho
logecho "Exceeded maximum argument limit of $args!"
logecho
$PROG -?
logecho
common::exit 1
fi
}
###############################################################################
# Get the md5 hash of a file
# @param file - The file
# @print the md5 hash
common::md5 () {
local file=$1
if which md5 >/dev/null 2>&1; then
md5 -q "$file"
else
md5sum "$file" | awk '{print $1}'
fi
}
###############################################################################
# Get the SHA hash of a file
# @param file - The file
# @param algo - Algorithm 1, 224, 256 (default), 384, 512, 512224, 512256
# output_type - Specifies output for the SHA:
# hash (default): outputs just the SHA
# file: outputs the SHA, two spaces, and the basename of the file
# @print the sha hash
common::sha () {
local file=$1
local algo=${2:-256}
local output_type=${3:-hash}
local shasum_output
which shasum >/dev/null 2>&1 || return 1
shasum_output=$(shasum -a"$algo" "$file")
if [[ "$output_type" != "full" ]]; then
echo "$shasum_output" | awk '{print $1}'
else
echo "$shasum_output" | sed 's/ .*\// /'
fi
}
###############################################################################
# Set the global GSUTIL and GCLOUD binaries
# Returns:
# 0 if both GSUTIL and GCLOUD are set to executables
# 1 if both GSUTIL and GCLOUD are not set to executables
common::set_cloud_binaries () {
logecho -n "Checking/setting cloud tools: "
for GSUTIL in "$(which gsutil)" /opt/google/google-cloud-sdk/bin/gsutil; do
if [[ -x $GSUTIL ]]; then
break
fi
done
for GCLOUD in "${GSUTIL/gsutil/gcloud}" "$(which gcloud)"; do
if [[ -x $GCLOUD ]]; then
break
fi
done
if [[ -x "$GSUTIL" && -x "$GCLOUD" ]]; then
logecho -r $OK
else
logecho -r $FAILED
return 1
fi
# 'gcloud docker' access is now set in .docker/config.json
# TODO: Reactivate when the deprecated functionality's replacement is working
# See deprecated bit in lib/releaselib.sh ($GCLOUD docker -- push)
#logrun $GCLOUD --quiet auth configure-docker || return 1
return 0
}
# right thing in common::trapclean().
common::trap common::trapclean ERR SIGINT SIGQUIT SIGTERM SIGHUP
# parse cmdline
common::namevalue "$@"
# Run common::manpage to show usage and man pages
common::manpage "$@"
# Set a TMPDIR
TMPDIR="${FLAGS_tmpdir:-/tmp}"
mkdir -p $TMPDIR
# Set some values that depend on $TMPDIR
PROGSTATE=$TMPDIR/$PROG-runstate
LOCAL_CACHE="$TMPDIR/buildresults-cache.$$"
###############################################################################
# gitlib.sh: GIT-related constants and functions
###############################################################################
###############################################################################
# CONSTANTS
###############################################################################
GITHUB_TOKEN=${FLAGS_github_token:-$GITHUB_TOKEN}
# This is a safety measure to trim extraneous leading/trailing whitespace from
# the OAuth token provided. Otherwise, authentication to GitHub will fail here.
GITHUB_TOKEN="$( echo "$GITHUB_TOKEN" | tr -d '[:space:]' )"
[[ -n "$GITHUB_TOKEN" ]] && GITHUB_TOKEN_FLAG=("-u" "${GITHUB_TOKEN}:x-oauth-basic")
GHCURL="curl -s --fail --retry 10 ${GITHUB_TOKEN_FLAG[*]}"
JCURL="curl -g -s --fail --retry 10"
GITHUB_API='https://api.github.com'
GITHUB_API_GRAPHQL="${GITHUB_API}/graphql"
K8S_GITHUB_API_ROOT="${GITHUB_API}/repos"
K8S_GITHUB_API="$K8S_GITHUB_API_ROOT/kubernetes/kubernetes"
K8S_GITHUB_RAW_ORG='https://raw.githubusercontent.com/kubernetes'
K8S_GITHUB_SEARCHAPI_ROOT='https://api.github.com/search/issues?per_page=100'
K8S_GITHUB_SEARCHAPI="$K8S_GITHUB_SEARCHAPI_ROOT&q=is:pr%20repo:kubernetes/kubernetes%20"
K8S_GITHUB_URL='https://github.com/kubernetes/kubernetes'
if ((FLAGS_gcb)); then
K8S_GITHUB_AUTH_ROOT="https://git@github.com/"
else
# ssh
K8S_GITHUB_AUTH_ROOT="git@github.com:"
fi
K8S_GITHUB_AUTH_URL="${K8S_GITHUB_AUTH_ROOT}kubernetes/kubernetes.git"
# Regular expressions for bash regex matching
# 0=entire branch name
# 1=Major
# 2=Minor
# 3=.Patch
# 4=Patch
BRANCH_REGEX="master|release-([0-9]{1,})\.([0-9]{1,})(\.([0-9]{1,}))*$"
# release - 1=Major, 2=Minor, 3=Patch, 4=-(alpha|beta|rc), 5=rev
# dotzero - 1=Major, 2=Minor
# build - 1=build number, 2=sha1
declare -A VER_REGEX=([release]="v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[a-zA-Z0-9]+)*\.*(0|[1-9][0-9]*)?"
[dotzero]="v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.0$"
[build]="([0-9]{1,})\+([0-9a-f]{5,40})"
)
###############################################################################
# releaselib.sh
###############################################################################
###############################################################################
# CONSTANTS
###############################################################################
# TODO(vdf): Need to reference K8s Infra projects here
readonly DEFAULT_PROJECT="kubernetes-release-test"
readonly PROD_PROJECT="kubernetes-release"
readonly TEST_PROJECT="kubernetes-release-test"
# TODO(vdf): Need to reference K8s Infra buckets here
readonly DEFAULT_BUCKET="kubernetes-release-gcb"
readonly PROD_BUCKET="kubernetes-release"
readonly TEST_BUCKET="kubernetes-release-gcb"
readonly CI_BUCKET="k8s-release-dev"
readonly GCRIO_PATH_PROD="registry.k8s.io"
readonly GCRIO_PATH_STAGING="gcr.io/k8s-staging-kubernetes"
readonly GCRIO_PATH_MOCK="${GCRIO_PATH_STAGING}/mock"
readonly KUBE_CROSS_REGISTRY="${GCRIO_PATH_PROD}/build-image"
readonly KUBE_CROSS_IMAGE="${KUBE_CROSS_REGISTRY}/kube-cross"
readonly KUBE_CROSS_CONFIG_LOCATION="build/build-image/cross"
# Set a globally usable variable for the changelog directory since we've been
# piecemeal search/replace-ing this and missing some cases.
readonly CHANGELOG_DIR="CHANGELOG"
###############################################################################
# Check that the GCS bucket exists and is writable.
#
# @param bucket - The gs release bucket name
# @return 1 if bucket does not exist or is not writable.
release::gcs::check_release_bucket() {
local bucket=$1
local tempfile=$TMPDIR/$PROG-gcs-write.$$
if ! $GSUTIL ls "gs://$bucket" >/dev/null 2>&1 ; then
logecho "Google Cloud Storage bucket does not exist: $bucket. Create the bucket with this command:"
logecho "$GSUTIL mb -p \"$GCLOUD_PROJECT\" \"gs://$bucket\""
logecho "If the bucket should be publicly readable, make it so with this command:"
logecho "$GSUTIL defacl ch -u AllUsers:R \"gs://$bucket\""
logecho "WARNING: This affects all objects uploaded to the bucket!"
return 1
fi
logecho -n "Checking write access to bucket $bucket: "
if logrun touch $tempfile && \
logrun $GSUTIL cp $tempfile gs://$bucket && \
logrun $GSUTIL rm gs://$bucket/${tempfile##*/} && \
logrun rm -f $tempfile; then
logecho $OK
else
logecho "$FAILED: You do not have access/write permission on $bucket." \
"Unable to continue."
return 1
fi
}
###############################################################################
# Creates a tarball for upload to a GCS staging directory.
# @param gcs_stage - the staging directory
# @param source and destination arguments
# @return 1 if tar directory or tarball creation fails
release::gcs::prepare_tarball() {
local gcs_stage=$1
shift
local src
local srcdir
local srcthing
local args
local split
local srcs
local dst
# Split the args into srcs... and dst
local args=("$@")
local split=$((${#args[@]}-1)) # Split point for src/dst args
local srcs=("${args[@]::$split}" )
local dst="${args[$split]}"
logrun mkdir -p $gcs_stage/$dst || return 1
for src in ${srcs[@]}; do
srcdir=$(dirname $src)
srcthing=$(basename $src)
tar c -C $srcdir $srcthing | tar x -C $gcs_stage/$dst || return 1
done
}
###############################################################################
# Ensure the destination bucket path doesn't already exist
# @param gcs_destination - GCS destination directory
# @return 1 on failure or if GCS destination already exists
# and --allow_dup does not set
release::gcs::destination_empty() {
local gcs_destination=$1
logecho -n "Checking whether $gcs_destination already exists: "
if $GSUTIL ls $gcs_destination >/dev/null 2>&1 ; then
logecho "- Destination exists. To remove, run:"
logecho " gsutil -m rm -r $gcs_destination"
if ((FLAGS_allow_dup)) ; then
logecho "flag --allow-dup set, continue with overwriting"
else
logecho "$FAILED"
return 1
fi
fi
logecho "$OK"
}
###############################################################################
# Push the release artifacts to GCS
# @param src - Source path
# @param dest - Destination path
# @return 1 on failure
release::gcs::push_release_artifacts() {
local src=$1
local dest=$2
logecho "Publish public release artifacts..."
# No need to check this for mock or stage runs
# Overwriting is ok
if ((FLAGS_nomock)) && ! ((FLAGS_stage)); then
release::gcs::destination_empty $dest || return 1
fi
# Copy the main set from staging to destination
# We explicitly don't set an ACL in the cp call, since doing so will override
# any default bucket ACLs.
logecho -n "- Copying artifacts to $dest: "
logrun -s $GSUTIL -qm cp -rc $src/* $dest/ || return 1
# This small sleep gives the eventually consistent GCS bucket listing a chance
# to stabilize before the diagnostic listing. There's no way to directly
# query for consistency, but it's OK if something is dropped from the
# debugging output.
sleep 5
logecho -n "- Listing final contents to log file: "
logrun -s $GSUTIL ls -lhr "$dest" || return 1
}
###############################################################################
# Locally stage the release artifacts to staging directory
# @param version - The version
# @param build_output - build output directory
# @optparam release_kind - defaults to kubernetes
# @return 1 on failure
release::gcs::locally_stage_release_artifacts() {
local version=$1
local build_output=$2
# --release-kind used by push-build.sh
local release_kind=${3:-"kubernetes"}
local platform
local release_stage=$build_output/release-stage
local release_tars=$build_output/release-tars
local gcs_stage=$build_output/gcs-stage/$version
local configure_vm
local src
local dst
logecho "Locally stage release artifacts..."
logrun rm -rf $gcs_stage || return 1
logrun mkdir -p $gcs_stage || return 1
# Stage everything in release directory
logecho "- Staging locally to ${gcs_stage##$build_output/}..."
release::gcs::prepare_tarball $gcs_stage $release_tars/* . || return 1
# Controls whether we publish md5 / sha1; clear to prevent publishing
# Although we would like to encourage users to stop using md5 / sha1,
# because all releases are published by the master branch, this broke
# anyone that was verifying the sha1 hash, including in CI builds.
# kops caught this in e.g. https://prow.k8s.io/view/gcs/kubernetes-jenkins/pr-logs/pull/kops/7462/pull-kops-e2e-kubernetes-aws/1164899358911500292/
# Instead we need to stop publishing sha1 and md5 on a release boundary.
local publish_old_hashes="1"
if [[ "$release_kind" == "kubernetes" ]]; then
if ! [[ $version =~ ${VER_REGEX[release]} ]]; then
logecho -r "$FAILED"
logecho "* Invalid version format! $version"
return 1
fi