From 63d2f5c4b19e26561af69ce9b6b6872562278d22 Mon Sep 17 00:00:00 2001 From: Keeley Hammond Date: Thu, 5 May 2022 15:22:19 -0700 Subject: [PATCH 1/2] build: remove S3 uploads --- .circleci/config.yml | 1 + script/lib/azput.js | 7 +--- script/lib/config.py | 11 ----- script/lib/s3put.js | 40 ------------------- script/lib/util.py | 21 +--------- script/release/release.js | 11 ----- script/release/uploaders/upload-index-json.py | 2 +- .../uploaders/upload-node-checksums.py | 2 +- .../release/uploaders/upload-node-headers.py | 10 ++--- script/release/uploaders/upload-symbols.py | 4 +- script/release/uploaders/upload.py | 5 ++- 11 files changed, 16 insertions(+), 98 deletions(-) delete mode 100644 script/lib/s3put.js diff --git a/.circleci/config.yml b/.circleci/config.yml index adcf9e502490c..b7556c5783095 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,6 +14,7 @@ parameters: type: boolean default: false + # TODO (vertedinde): migrate this variable to upload-to-az upload-to-s3: type: string default: '1' diff --git a/script/lib/azput.js b/script/lib/azput.js index 0e6a75cd2ad8f..6abd0220db768 100644 --- a/script/lib/azput.js +++ b/script/lib/azput.js @@ -20,12 +20,7 @@ let anErrorOccurred = false; function next (done) { const file = files.shift(); if (!file) return done(); - let key = filenameToKey(file); - // TODO: When we drop s3put, migrate the key to not include atom-shell in the callsites - key = key.replace('atom-shell/dist/', 'headers/dist/'); - key = key.replace('atom-shell/symbols/', 'symbols/'); - key = key.replace('atom-shell/tmp/', 'checksums-scratchpad/'); - key = key.replace('electron-artifacts/', 'release-builds/'); + const key = filenameToKey(file); const [containerName, ...keyPath] = key.split('/'); const blobKey = keyPath.join('/'); diff --git a/script/lib/config.py b/script/lib/config.py index 9ef44a43fe3d9..181f2b621bd54 100644 --- a/script/lib/config.py +++ b/script/lib/config.py @@ -53,17 +53,6 @@ def get_env_var(name): return value -def s3_config(): - config = (get_env_var('S3_BUCKET'), - get_env_var('S3_ACCESS_KEY'), - get_env_var('S3_SECRET_KEY')) - message = ('Error: Please set the $ELECTRON_S3_BUCKET, ' - '$ELECTRON_S3_ACCESS_KEY, and ' - '$ELECTRON_S3_SECRET_KEY environment variables') - assert all(len(c) for c in config), message - return config - - def enable_verbose_mode(): print('Running in verbose mode') global verbose_mode diff --git a/script/lib/s3put.js b/script/lib/s3put.js deleted file mode 100644 index 8f92b6d95decf..0000000000000 --- a/script/lib/s3put.js +++ /dev/null @@ -1,40 +0,0 @@ -/* eslint-disable camelcase */ -const AWS = require('aws-sdk'); -const fs = require('fs'); -const path = require('path'); -AWS.config.update({ region: 'us-west-2' }); -const s3 = new AWS.S3({ apiVersion: '2006-03-01' }); - -const args = require('minimist')(process.argv.slice(2)); - -let { bucket, prefix = '/', key_prefix = '', grant, _: files } = args; -if (prefix && !prefix.endsWith(path.sep)) prefix = path.resolve(prefix) + path.sep; - -function filenameToKey (file) { - file = path.resolve(file); - if (file.startsWith(prefix)) file = file.substr(prefix.length - 1); - return key_prefix + (path.sep === '\\' ? file.replace(/\\/g, '/') : file); -} - -let anErrorOccurred = false; -function next (done) { - const file = files.shift(); - if (!file) return done(); - const key = filenameToKey(file); - console.log(`Uploading '${file}' to bucket '${bucket}' with key '${key}'...`); - s3.upload({ - Bucket: bucket, - Key: key, - Body: fs.createReadStream(file), - ACL: grant - }, (err, data) => { - if (err) { - console.error(err); - anErrorOccurred = true; - } - next(done); - }); -} -next(() => { - process.exit(anErrorOccurred ? 1 : 0); -}); diff --git a/script/lib/util.py b/script/lib/util.py index 0ae5c2b284db3..7ef7b953e262a 100644 --- a/script/lib/util.py +++ b/script/lib/util.py @@ -20,7 +20,7 @@ from urllib2 import urlopen import zipfile -from lib.config import is_verbose_mode, s3_config +from lib.config import is_verbose_mode ELECTRON_DIR = os.path.abspath( os.path.dirname(os.path.dirname(os.path.dirname(__file__))) @@ -161,26 +161,9 @@ def get_electron_version(): return 'v' + f.read().strip() def store_artifact(prefix, key_prefix, files): - # Legacy S3 Bucket - s3put(prefix, key_prefix, files) - # New AZ Storage + # Azure Storage azput(prefix, key_prefix, files) -def s3put(prefix, key_prefix, files): - bucket, access_key, secret_key = s3_config() - env = os.environ.copy() - env['AWS_ACCESS_KEY_ID'] = access_key - env['AWS_SECRET_ACCESS_KEY'] = secret_key - output = execute([ - 'node', - os.path.join(os.path.dirname(__file__), 's3put.js'), - '--bucket', bucket, - '--prefix', prefix, - '--key_prefix', key_prefix, - '--grant', 'public-read', - ] + files, env) - print(output) - def azput(prefix, key_prefix, files): env = os.environ.copy() output = execute([ diff --git a/script/release/release.js b/script/release/release.js index 52dcdbb562305..38683ee7a6733 100755 --- a/script/release/release.js +++ b/script/release/release.js @@ -78,8 +78,6 @@ async function validateReleaseAssets (release, validatingRelease) { console.log(`${fail} error verifyingShasums`, err); }); } - const s3RemoteFiles = s3RemoteFilesForVersion(release.tag_name); - await verifyShasumsForRemoteFiles(s3RemoteFiles, true); const azRemoteFiles = azRemoteFilesForVersion(release.tag_name); await verifyShasumsForRemoteFiles(azRemoteFiles, true); } @@ -196,15 +194,6 @@ const cloudStoreFilePaths = (version) => [ 'SHASUMS256.txt' ]; -function s3RemoteFilesForVersion (version) { - const bucket = 'https://gh-contractor-zcbenz.s3.amazonaws.com/'; - const versionPrefix = `${bucket}atom-shell/dist/${version}/`; - return cloudStoreFilePaths(version).map((filePath) => ({ - file: filePath, - url: `${versionPrefix}${filePath}` - })); -} - function azRemoteFilesForVersion (version) { const azCDN = 'https://artifacts.electronjs.org/headers/'; const versionPrefix = `${azCDN}dist/${version}/`; diff --git a/script/release/uploaders/upload-index-json.py b/script/release/uploaders/upload-index-json.py index fe82e24716ac9..29441b3dce0ac 100755 --- a/script/release/uploaders/upload-index-json.py +++ b/script/release/uploaders/upload-index-json.py @@ -59,7 +59,7 @@ def main(): with open(index_json, "wb") as f: f.write(new_content) - store_artifact(OUT_DIR, 'atom-shell/dist', [index_json]) + store_artifact(OUT_DIR, 'headers/dist/', [index_json]) if __name__ == '__main__': diff --git a/script/release/uploaders/upload-node-checksums.py b/script/release/uploaders/upload-node-checksums.py index dfa8ef3595ad7..7ef6d7a738821 100755 --- a/script/release/uploaders/upload-node-checksums.py +++ b/script/release/uploaders/upload-node-checksums.py @@ -29,7 +29,7 @@ def main(): ] if args.target_dir is None: - store_artifact(directory, 'atom-shell/dist/{0}'.format(args.version), + store_artifact(directory, 'headers/dist/{0}'.format(args.version), checksums) else: copy_files(checksums, args.target_dir) diff --git a/script/release/uploaders/upload-node-headers.py b/script/release/uploaders/upload-node-headers.py index 862eea96e8b02..6b21c5b5e8ea8 100755 --- a/script/release/uploaders/upload-node-headers.py +++ b/script/release/uploaders/upload-node-headers.py @@ -45,9 +45,9 @@ def upload_node(version): versioned_header_tar = header_tar.format(version) shutil.copy2(generated_tar, os.path.join(GEN_DIR, versioned_header_tar)) - store_artifact(GEN_DIR, 'atom-shell/dist/{0}'.format(version), + store_artifact(GEN_DIR, 'headers/dist/{0}'.format(version), glob.glob('node-*.tar.gz')) - store_artifact(GEN_DIR, 'atom-shell/dist/{0}'.format(version), + store_artifact(GEN_DIR, 'headers/dist/{0}'.format(version), glob.glob('iojs-*.tar.gz')) if PLATFORM == 'win32': @@ -73,13 +73,13 @@ def upload_node(version): shutil.copy2(electron_lib, v4_node_lib) # Upload the node.lib. - store_artifact(DIST_DIR, 'atom-shell/dist/{0}'.format(version), [node_lib]) + store_artifact(DIST_DIR, 'headers/dist/{0}'.format(version), [node_lib]) # Upload the iojs.lib. - store_artifact(DIST_DIR, 'atom-shell/dist/{0}'.format(version), [iojs_lib]) + store_artifact(DIST_DIR, 'headers/dist/{0}'.format(version), [iojs_lib]) # Upload the v4 node.lib. - store_artifact(DIST_DIR, 'atom-shell/dist/{0}'.format(version), + store_artifact(DIST_DIR, 'headers/dist/{0}'.format(version), [v4_node_lib]) diff --git a/script/release/uploaders/upload-symbols.py b/script/release/uploaders/upload-symbols.py index 3e877e64772db..04e2def3b7808 100755 --- a/script/release/uploaders/upload-symbols.py +++ b/script/release/uploaders/upload-symbols.py @@ -56,7 +56,7 @@ def main(): files += glob.glob(SYMBOLS_DIR + '/*/*/*.src.zip') - # The file upload needs to be atom-shell/symbols/:symbol_name/:hash/:symbol + # The file upload needs to be symbols/:symbol_name/:hash/:symbol os.chdir(SYMBOLS_DIR) files = [os.path.relpath(f, os.getcwd()) for f in files] @@ -84,7 +84,7 @@ def run_symstore(pdb, dest, product): def upload_symbols(files): - store_artifact(SYMBOLS_DIR, 'atom-shell/symbols', + store_artifact(SYMBOLS_DIR, 'symbols', files) diff --git a/script/release/uploaders/upload.py b/script/release/uploaders/upload.py index 26191bc4780fa..d49e0a5f936bb 100755 --- a/script/release/uploaders/upload.py +++ b/script/release/uploaders/upload.py @@ -333,8 +333,9 @@ def upload_electron(release, file_path, args): pass # if upload_to_s3 is set, skip github upload. + # todo (vertedinde): migrate this variable to upload_to_az if args.upload_to_s3: - key_prefix = 'electron-artifacts/{0}_{1}'.format(args.version, + key_prefix = 'release-builds/{0}_{1}'.format(args.version, args.upload_timestamp) store_artifact(os.path.dirname(file_path), key_prefix, [file_path]) upload_sha256_checksum(args.version, file_path, key_prefix) @@ -359,7 +360,7 @@ def upload_io_to_github(release, filename, filepath, version): def upload_sha256_checksum(version, file_path, key_prefix=None): checksum_path = '{}.sha256sum'.format(file_path) if key_prefix is None: - key_prefix = 'atom-shell/tmp/{0}'.format(version) + key_prefix = 'checksums-scratchpad/{0}'.format(version) sha256 = hashlib.sha256() with open(file_path, 'rb') as f: sha256.update(f.read()) From a566a6ba73eb8824d63c516e0ef2a773e7fb823b Mon Sep 17 00:00:00 2001 From: Keeley Hammond Date: Fri, 6 May 2022 12:57:18 -0700 Subject: [PATCH 2/2] build: remove ending slash upload-index-json (#34125) --- script/release/uploaders/upload-index-json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/release/uploaders/upload-index-json.py b/script/release/uploaders/upload-index-json.py index 29441b3dce0ac..9ed44d6eb797a 100755 --- a/script/release/uploaders/upload-index-json.py +++ b/script/release/uploaders/upload-index-json.py @@ -59,7 +59,7 @@ def main(): with open(index_json, "wb") as f: f.write(new_content) - store_artifact(OUT_DIR, 'headers/dist/', [index_json]) + store_artifact(OUT_DIR, 'headers/dist', [index_json]) if __name__ == '__main__':