diff --git a/Bundle-Debian/setup_builder.py b/Bundle-Debian/setup_builder.py index 6dc5fbcb..0356a317 100644 --- a/Bundle-Debian/setup_builder.py +++ b/Bundle-Debian/setup_builder.py @@ -19,7 +19,6 @@ deps += re.sub(versions_re, ' ', line) elif re.search(build_deps_re, line): in_deps = True - deps += re.sub(build_deps_re, '', re.sub(versions_re, ' ', line)) if in_deps and not re.search(continues_re, line): @@ -37,6 +36,6 @@ if len(arg) > 0: cleanargs.append(arg) -print ' '.join(cleanargs) +print(' '.join(cleanargs)) subprocess.call(cleanargs) diff --git a/Bundle-Mac/buildDMG.py b/Bundle-Mac/buildDMG.py index 2f9ac77b..4b95bd9b 100644 --- a/Bundle-Mac/buildDMG.py +++ b/Bundle-Mac/buildDMG.py @@ -6,14 +6,14 @@ import argparse import build_tools -#Most of the script is ripped out of the old jenkins.py from Bundle-Mac. +#Most of the script is ripped out of the old jenkins.py from Bundle-Mac. #It peforms the subset of duties for packageing the already built libraries. def add_platform_args(args): arg_parser = argparse.ArgumentParser( description='Script to create mac packages', parents=[build_tools.get_parent_parser()]) - + arg_parser.add_argument('--no-digitizer', action='store_true', help='disable digitizer in build (only works on mac os)') @@ -38,7 +38,7 @@ def change_sym(lib_path, sym_dest, bin_filename): subprocess.check_call(['install_name_tool', '-change', lib_path, sym_dest, bin_filename]) except subprocess.CalledProcessError as e: - print 'Error changing install name:' + str(e) + print("Error changing install name: %s" % e) raise def change_id(bin_dest, bin_filename): @@ -48,7 +48,7 @@ def change_id(bin_dest, bin_filename): try: subprocess.check_call(['install_name_tool', '-id', bin_dest, bin_filename]) except subprocess.CalledProcessError as e: - print 'Error changing install name:' + str(e) + print("Error changing install name: %s" % e) raise binmatch = re.compile('(Mach-O 64-bit executable x86_64|Mach-O 64-bit dynamically linked shared library x86_64)') @@ -139,7 +139,7 @@ def fix_lib_paths(fake_root, old_lib_paths): binaries = [] for binary in these_bins: - print "Fixing binary " + binary + print("Fixing binary %s" % binary) for old_lib_path in old_lib_paths: binaries += fix_binary(binary, old_lib_path, fake_root, os.path.join(fake_root, new_lib_path), @@ -187,10 +187,10 @@ def make_installer(package_dir, resources_dir, distribution_xml, outfile): '--resources', resources_dir, outfile]) except OSError as e: - print 'productbuild not found: ' + str(e) + print("productbuild not found: %s" % e) raise except subprocess.CalledProcessError as e: - print 'productbuild failed: ' + str(e) + print("productbuild failed: %s" % e) sys.exit(e.returncode) return outfile @@ -207,10 +207,10 @@ def make_uninstaller(workspace, outfile, resources_dir, scripts_dir): outfile]) except OSError as e: - print 'pkgbuild not found: ' + str(e) + print("pkgbuild not found: %s" % e) raise except subprocess.CalledProcessError as e: - print 'pkgbuild failed: ' + str(e) + print("pkgbuild failed: %s" % e) sys.exit(e.returncode) return outfile @@ -265,10 +265,10 @@ def make_dmg(workspace, package_dir, pkgname, version, resources_dir, no_digitiz '-ov', '-volname', pkgname, '-fs', 'HFS+', '-srcfolder', dmg_root] - print " ".join(cmd) + print(" ".join(cmd)) subprocess.check_call(cmd) except subprocess.CalledProcessError as e: - print 'hdiutil create failed: ' + str(e) + print("hdiutil create failed: %s" % e) sys.exit(e.returncode) final_dmg = os.path.join(workspace, versioned_name + '.dmg') @@ -280,7 +280,7 @@ def make_dmg(workspace, package_dir, pkgname, version, resources_dir, no_digitiz '-format', 'UDZO', '-o', final_dmg]) except subprocess.CalledProcessError as e: - print 'udiutil convert failed: ' + str(e) + print("udiutil convert failed: %s" % e) sys.exit(e.returncode) #shutil.rmtree(variant_dir) @@ -325,14 +325,14 @@ def make_packages(package_dir, core_pkg]) except OSError as e: - print 'pkgbuild not found: ' + str(e) + print("pkgbuild not found: %s" % e) raise except subprocess.CalledProcessError as e: - print 'pkgbuild failed: ' + str(e) + print("pkgbuild failed: %s" % e) sys.exit(e.returncode) def run(workspace, fake_root, version,additional_args=[]): - print("running buildDMG") + print("running buildDMG") #arguments = add_platform_args(additional_args) print('Copying EULA') shutil.copy(os.path.join(workspace,'Legal','EULA.txt'), diff --git a/Bundle-Mac/glue_qt.py b/Bundle-Mac/glue_qt.py deleted file mode 100644 index 4ecd5e90..00000000 --- a/Bundle-Mac/glue_qt.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/python - -import os -import sys -import shutil -import subprocess -import re - -def create_path(path): - (head, tail) = os.path.split(path) - - if not os.path.exists(head): - create_path(head) - - if not os.path.exists(path): - os.mkdir(path) - -qt64 = os.path.join('/', 'usr', 'local', 'Qt-5.1.1') -qt32 = os.path.join('/', 'usr', 'local', 'Qt-5.1.1-32') - -qt64bak = os.path.join('/', 'tmp', 'Qt-5.1.1-64-orig') - -qtjoined = os.path.join('/', 'tmp', 'Qt-5.1.1-joined') - -if os.path.exists(qt64bak): - shutil.rmtree(qt64bak) - -shutil.copytree(qt64, qt64bak) - -if os.path.exists(qtjoined): - shutil.rmtree(qtjoined) - - -base = re.compile('^' + qt64) -binmatch = re.compile('(Mach-O 64-bit executable x86_64|Mach-O 64-bit dynamically linked shared library x86_64)') - -for root64, subdirs, filenames in os.walk(qt64): - root32 = base.sub(qt32, root64) - rootjoined = base.sub(qtjoined, root64) - create_path(rootjoined) - - for filename in filenames: - file64 = os.path.join(root64, filename) - - if os.path.islink(file64): - continue - - filetype = subprocess.Popen(['file', file64], - stdout=subprocess.PIPE) - found = False - - for line in filetype.stdout.readlines(): - if binmatch.search(line) is not None: - found = True - break - - if not found: continue - - print 'found file ' + file64 - - file32 = os.path.join(root32, filename) - - if not os.path.exists(file32): - print '32 bit equivalent missing' - continue - - filejoined = os.path.join(rootjoined, filename) - subprocess.check_call(['lipo', file64, file32, '-create', '-output', filejoined]) - shutil.copyfile(filejoined, file64) diff --git a/Bundle-Mac/jenkins.py b/Bundle-Mac/jenkins.py index 17dd4386..357ee2ec 100644 --- a/Bundle-Mac/jenkins.py +++ b/Bundle-Mac/jenkins.py @@ -30,7 +30,7 @@ def change_sym(lib_path, sym_dest, bin_filename): subprocess.check_call(['install_name_tool', '-change', lib_path, sym_dest, bin_filename]) except subprocess.CalledProcessError as e: - print 'Error changing install name:' + str(e) + print("Error changing install name: %s" % e) raise @@ -45,7 +45,7 @@ def change_id(bin_dest, bin_filename): bin_dest, bin_filename]) except subprocess.CalledProcessError as e: - print 'Error changing install name:' + str(e) + print("Error changing install name: %s" % e) raise @@ -141,7 +141,7 @@ def fix_lib_paths(fake_root, old_lib_paths): binaries = [] for binary in these_bins: - print "Fixing binary " + binary + print("Fixing binary %s" % binary) for old_lib_path in old_lib_paths: binaries += fix_binary(binary, old_lib_path, fake_root, os.path.join(fake_root, new_lib_path), @@ -203,10 +203,10 @@ def make_uninstaller(workspace, outfile, resources_dir, scripts_dir): outfile]) except OSError as e: - print 'pkgbuild not found: ' + str(e) + print("pkgbuild not found: %s" % e) return build_tools.builders.FAILURE except subprocess.CalledProcessError as e: - print 'pkgbuild failed: ' + str(e) + print("pkgbuild failed: %s" % e) return build_tools.builders.FAILURE return build_tools.builders.SUCCESS @@ -256,10 +256,10 @@ def make_packages(package_dir, core_pkg]) except OSError as e: - print 'pkgbuild not found: ' + str(e) + print("pkgbuild not found: %s" % e) return build_tools.builders.FAILURE except subprocess.CalledProcessError as e: - print 'pkgbuild failed: ' + str(e) + print("pkgbuild failed: " % e) return build_tools.builders.FAILURE return build_tools.builders.SUCCESS @@ -274,10 +274,10 @@ def make_installer(package_dir, resources_dir, distribution_xml, outfile): '--resources', resources_dir, outfile]) except OSError as e: - print 'productbuild not found: ' + str(e) + print("productbuild not found: %s" % e) return build_tools.builders.FAILURE except subprocess.CalledProcessError as e: - print 'productbuild failed: ' + str(e) + print("productbuild failed: %s" % e) return build_tools.builders.FAILURE return build_tools.builders.SUCCESS @@ -355,10 +355,10 @@ def make_dmg( # http://apple.stackexchange.com/questions/156994 '-megabytes', '400'] - print " ".join(cmd) + print(" ".join(cmd)) subprocess.check_call(cmd) except subprocess.CalledProcessError as e: - print 'hdiutil create failed: ' + str(e) + print("hdiutil create failed: %s" % e) return build_tools.builders.FAILURE final_dmg = os.path.join(workspace, versioned_name + '.dmg') @@ -370,7 +370,7 @@ def make_dmg( '-format', 'UDZO', '-o', final_dmg]) except subprocess.CalledProcessError as e: - print 'udiutil convert failed: ' + str(e) + print("udiutil convert failed: %s" % e) return build_tools.builders.FAILURE shutil.rmtree(variant_dir) diff --git a/Bundle-Mac/sign_install_dmg.py b/Bundle-Mac/sign_install_dmg.py index 7cc8fa5d..e7641290 100644 --- a/Bundle-Mac/sign_install_dmg.py +++ b/Bundle-Mac/sign_install_dmg.py @@ -12,7 +12,7 @@ dmg_size = os.path.getsize(orig_dmg) >> 20 # add a few MB of buffer dmg_size += 10 -print ("DMG size: ", dmg_size) +print("DMG size: %s" % dmg_size) hdiutil = subprocess.Popen(["hdiutil", "attach", orig_dmg], stdout=subprocess.PIPE) diff --git a/Bundle-Redhat/installRPMS.py b/Bundle-Redhat/installRPMS.py index 8869010c..7fd52792 100644 --- a/Bundle-Redhat/installRPMS.py +++ b/Bundle-Redhat/installRPMS.py @@ -33,7 +33,7 @@ installed.append(pkgname) args = ['sudo', 'rpm', '-Uvh'] args.extend(tprpms) - print ' '.join(args) + print(" ".join(args)) subprocess.check_call(args) finally: - print(installed) \ No newline at end of file + print(installed) diff --git a/Bundle-Redhat/sign_rpms.py b/Bundle-Redhat/sign_rpms.py index 366799f3..8d0832f0 100644 --- a/Bundle-Redhat/sign_rpms.py +++ b/Bundle-Redhat/sign_rpms.py @@ -38,7 +38,7 @@ rpmdirs[root] = 1 if len(sys.argv) > 5: if sys.argv[5]=='-nosign': - print "not signing" + print("not signing") else: cmd = ['rpm', '--addsign'] cmd.extend(rpms) diff --git a/Bundle-Win/jenkins.py b/Bundle-Win/jenkins.py index 0b792d93..cb0e4a81 100644 --- a/Bundle-Win/jenkins.py +++ b/Bundle-Win/jenkins.py @@ -17,13 +17,13 @@ def zip_argo_driver(argo_driver_folder, zipname): - print "Creating zip of Argo driver" + print("Creating zip of Argo driver") argo_zip = zipfile.ZipFile(zipname, 'w') for root, dirs, files in os.walk(argo_driver_folder): for file in files: path = os.path.join(root, file) path_minus_argo_root = path[len(argo_driver_folder)+1:] - print " Writing " + path_minus_argo_root + " to " + zipname + print("Writing %s to %s" % (path_minus_argo_root, zipname) argo_zip.write(path, path_minus_argo_root) argo_zip.close() @@ -33,7 +33,7 @@ def make_installer(workspace, env, version, is_win64, argo): if not os.path.exists(nsis_exe): nsis_exe = 'C:\\Program Files (x86)\\NSIS\\makensis.exe' if not os.path.exists(nsis_exe): - print 'No NSIS detected. failing' + print("No NSIS detected. failing") return build_tools.builders.FAILURE nsis_script = os.path.join(workspace, 'Bundle-Win', 'makerbot.nsi') @@ -55,11 +55,11 @@ def make_installer(workspace, env, version, is_win64, argo): '/DARGO=true', '/DARGO_ROOT=' + build_tools.source_list.argo_v3driver.dir] nsis_command += [nsis_script] - print ' '.join(nsis_command) + print(" ".join(nsis_command)) try: subprocess.check_call(nsis_command, env=env, stdout=sys.stdout) except subprocess.CalledProcessError as e: - print 'nsis script failed: ' + str(e) + print("nsis script failed: %s" % e) return build_tools.builders.FAILURE return build_tools.builders.SUCCESS diff --git a/Bundle-Win/sign_installer.py b/Bundle-Win/sign_installer.py index f458581e..a4280669 100644 --- a/Bundle-Win/sign_installer.py +++ b/Bundle-Win/sign_installer.py @@ -5,6 +5,6 @@ os.environ['PATH'] += ';c:\\Program Files\\Microsoft SDKs\\Windows\\v7.1\\Bin' os.environ['PATH'] += ';c:\\Program Files (x86)\\Windows Kits\\8.1\\bin\\x86' -print os.environ['PATH'] +print(os.environ['PATH']) subprocess.call(['signtool', 'sign', '/a', '/t', 'http://timestamp.digicert.com', sys.argv[1]]) diff --git a/amazon_s3_tools/logcheck.py b/amazon_s3_tools/logcheck.py index e9c7ed7e..6e9ca63a 100755 --- a/amazon_s3_tools/logcheck.py +++ b/amazon_s3_tools/logcheck.py @@ -6,12 +6,12 @@ import uuid def main(baseDir='.'): - """ Tool for scanning Amazon s3 logs, to check for what software + """ Tool for scanning Amazon s3 logs, to check for what software was downloaded recently, by whom. This is used to generate download stats from our website to get a rough idea of what OS and when people are downloading updates to MakerWare or other programs """ - download numbers/stats on + download numbers/stats on logRegex = re.compile('access\d{4}(-\d{2}){5}-\w{16}') files = os.listdir(baseDir) if baseDir is not None else [] filesLen = len(files) @@ -19,10 +19,10 @@ def main(baseDir='.'): print("non-log files removed: %d" %(filesLen- len(files))) print("sorting by date") files.sort() #default sort is good enough - - dataDict = {} + + dataDict = {} for f in files: - data = yank_data(f) + data = yank_data(f) dataDict.update(data) stats = {} @@ -35,7 +35,7 @@ def main(baseDir='.'): if k2 == 'r_datetime': continue #skip datetime for now if k2 not in stats: - stats[k2] = {} + stats[k2] = {} statSubDict = stats[k2] if propDict[k2] not in statSubDict: statSubDict[propDict[k2]] = 0 @@ -57,46 +57,46 @@ def main(baseDir='.'): cntList = [stats['r_resource'][req] for req in winReq] winStat = reduce(lambda a,b: a+b, cntList) print('Windows download count: %s ',str(winStat)) - + cntList = [stats['r_resource'][req] for req in pkgReq] winStat = reduce(lambda a,b: a+b, cntList) print('Mac download count: %s ',str(winStat)) - + cntList = [stats['r_resource'][req] for req in preciseReq] winStat = reduce(lambda a,b: a+b, cntList) print('Ubuntu Precise download count: %s ',str(winStat)) - + cntList = [stats['r_resource'][req] for req in oneiricReq] winStat = reduce(lambda a,b: a+b, cntList) print('Ubuntu Oneiric download count: %s ',str(winStat)) - + print("## Downloads by Version") dl_files = [] # extend lists dl_files.extend(winReq) dl_files.extend(pkgReq) dl_files.extend(preciseReq) dl_files.extend(oneiricReq) - - ver_101_req = [s for s in dl_files if re.match('.*makerware_1\.0\.1',s)] + + ver_101_req = [s for s in dl_files if re.match('.*makerware_1\.0\.1',s)] cntList = [stats['r_resource'][req] for req in ver_101_req] stats101 = reduce(lambda a,b: a+b, cntList) print('Version 1.0.1 download count: %s ',str(stats101)) for x in ver_101_req: - dl_files.remove(x) + dl_files.remove(x) - ver_100_req = [s for s in dl_files if re.match('.*makerware_1\.00',s)] + ver_100_req = [s for s in dl_files if re.match('.*makerware_1\.00',s)] cntList = [stats['r_resource'][req] for req in ver_100_req] stats100 = reduce(lambda a,b: a+b, cntList) print('Version 1.0 download count: %s ',str(stats100)) for x in ver_100_req: - dl_files.remove(x) + dl_files.remove(x) - ver_006_req = [s for s in dl_files if re.match('.*makerware_0\.06',s)] + ver_006_req = [s for s in dl_files if re.match('.*makerware_0\.06',s)] cntList = [stats['r_resource'][req] for req in ver_006_req] stats006 = reduce(lambda a,b: a+b, cntList) print('Version 0.6 download count: %s ',str(stats006)) for x in ver_006_req: - dl_files.remove(x) + dl_files.remove(x) cntList = [stats['r_resource'][req] for req in dl_files] statsOther = reduce(lambda a,b: a+b, cntList) @@ -108,7 +108,7 @@ def main(baseDir='.'): def get_first_a3data_split(sp=[]): """ connvert first slice of s3 data into a readable format retDict['r_site'] - request name of site string - retDict['r_datetime'] - request datetime obj + retDict['r_datetime'] - request datetime obj """ retDict = {} if len(sp) < 9: @@ -120,7 +120,7 @@ def get_first_a3data_split(sp=[]): dtString = ' '.join(sp[2:3]).strip('[]') retDict['r_datetime'] = datetime.strptime(dtString,'%d/%b/%Y:%H:%M:%S') #ignore sp[3] , timezone, it just fils - retDict['r_req_ip'] = sp[4] #ip addr of Requestor? + retDict['r_req_ip'] = sp[4] #ip addr of Requestor? #ignore sp[5], '-' and sp[6] 'WEBSITE.GET.OBJECT' retDict['r_resource'] = sp[8] #ignore sp[9] as '' @@ -128,17 +128,17 @@ def get_first_a3data_split(sp=[]): def a3_log_to_dict(a3data=''): - """ takes a single a3 line of data, turns it into + """ takes a single a3 line of data, turns it into a dictionary and returns that """ retDict = {} if a3data is None or a3data is '': - return retDict + return retDict try: block = a3data.split('"') - if len(block) == 1: + if len(block) == 1: #no quoted string. Re-slice block by ' ' t_block = block[0].split(' ') - block = [] + block = [] block.append(' '.join(t_block[0:9])) block.append(' '.join(t_block[10:16])) block.append(' ') @@ -150,8 +150,8 @@ def a3_log_to_dict(a3data=''): request, get, get_status= block[0],block[1],block[2] src_link, requester, breakln= block[3],block[5],block[6] #blank = block[4] - if ( breakln != ' -\n'): - print ('breakline error in a3 data block %s' %str(block) ) + if ( breakln != ' -\n'): + print("breakline error in a3 data block %s" % block) return {} # skip it, we didn't write app the parsers sp = request.split(' ') retDict.update(get_first_a3data_split(sp)) @@ -169,11 +169,11 @@ def a3_log_to_dict(a3data=''): pdb.set_trace() return retDict - + def yank_data(filename): retDict = {} - if filename is not None: + if filename is not None: fh = open(filename, 'r') lines = fh.readlines() fh.close() diff --git a/amazon_s3_tools/ubuntu_release.py b/amazon_s3_tools/ubuntu_release.py index 2e182e8b..45012158 100644 --- a/amazon_s3_tools/ubuntu_release.py +++ b/amazon_s3_tools/ubuntu_release.py @@ -12,6 +12,15 @@ import s3_utils import argparse +# Assigning Temporary AWS credentials generated from samlapi.py script +# to Temporary Environmental Variables in the OS +# Remember to copy and paste the text commands generated +# from samlapi.py -e -u +# When running this script, be sure that the Environment Variables are on the same usergroup level +AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') +AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') +AWS_SESSION_TOKEN = os.environ.get('AWS_SESSION_TOKEN') + UBUNTU_INTERNAL_SIGNED_REPO_URL = 'http://sid.soft.makerbot.net/apt/signed' #S3_BUCKET should be changed to 'downloads-makerbot-com' at the time of release @@ -89,14 +98,15 @@ def main(args): if subprocess.call(WGET_COMMAND): log_and_print('ABORTING: wget failed to pull the files specified', log) exit() - + log_and_print('WGet took: ' + str((time.time() - wg_start)/60) + ' minutes', log) - print "\n At this point make sure that the downloaded ubuntu repo is " +\ - "ready for release (Check for tar.gz source files that shouldn't be there)\n" - raw_input("\nThen Press Enter\n") - raw_input("\nSecond Safety Check, Press Enter Again") + print("\nAt this point make sure that the downloaded ubuntu repo is ready " + "for release (Check for tar.gz source files that shouldn't be " + "there)\n") + raw_input("\nThen Press Enter\n")) + raw_input("\nSecond Safety Check, Press Enter Again")) #Create s3 connection and get the bucket bucket = s3_utils.get_s3_connection().Bucket(S3_BUCKET) t_linuxstart = time.time() diff --git a/artifactory-migration.py b/artifactory-migration.py index 69306d63..f0f5c52d 100644 --- a/artifactory-migration.py +++ b/artifactory-migration.py @@ -15,7 +15,7 @@ def download_file(path, fp): """ Download a single file specified by path to the local filelocation fp """ - print "Downloading file: " + fp + print("Downloading file: %" % fp) with path.open() as remote: with open(fp, 'wb') as local: local.write(remote.read()) diff --git a/build-repos.py b/build-repos.py index 1ff8d41a..1b03e3f5 100755 --- a/build-repos.py +++ b/build-repos.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python import argparse import os @@ -43,11 +43,11 @@ def build_repos(args): rv = repo.build(**build_args) if rv is build_tools.builders.FAILURE: - print('Building ' + repo.name + ' failed') + print("Building %s failed" % repo.name) if not args.keep_going: return rv if rv is build_tools.builders.FAILED_TESTS: - print(repo.name + ' failed while running tests') + print('%s failed while running tests' % repo.name) if not args.keep_going: return rv return build_tools.builders.SUCCESS diff --git a/build_tools/__init__.py b/build_tools/__init__.py index f16ec951..8879f2dc 100644 --- a/build_tools/__init__.py +++ b/build_tools/__init__.py @@ -11,26 +11,27 @@ def _delete_gist(py_file): base = os.path.join(_this_dir, py_file) for path in (base, base + 'c'): if os.path.exists(path): + print("deleting %s" % path) os.remove(path) _delete_gist('artifactory_utils.py') # Pretend artifactory_utils still lives here -from jenkins_tools import artifactory_utils -sys.modules['build_tools.artifactory_utils'] = artifactory_utils +# from jenkins_tools import artifactory_utils +# sys.modules['build_tools.artifactory_utils'] = artifactory_utils _delete_gist('mac_pkg_helpers.py') # Pretend mac_pkg_helpers still lives here from jenkins_tools import mac_pkg_helpers -sys.modules['build_tools.mac_pkg_helpers'] = mac_pkg_helpers +# sys.modules['build_tools.mac_pkg_helpers'] = mac_pkg_helpers -from common import * -from source import Source, Repo, Tag -from toolchain import Toolchain -import builders -import utils -import paths -import source -import toolchain -import source_list -import toolchain_list -import platforms +from .common import * +from .source import Source, Repo, Tag +from .toolchain import Toolchain +from . import builders +from . import utils +from . import paths +from . import source +from . import toolchain +from . import source_list +from . import toolchain_list +from . import platforms diff --git a/build_tools/builders.py b/build_tools/builders.py index 94f99b01..a4a6a706 100644 --- a/build_tools/builders.py +++ b/build_tools/builders.py @@ -17,12 +17,12 @@ import sys import copy -import mac_pkg_helpers -import paths -import platforms -import utils +from jenkins_tools import mac_pkg_helpers as mac_pkg_helpers +from jenkins_tools import artifactory_utils as artifactory_utils -import artifactory_utils +from . import paths +from . import platforms +from . import utils # Build outcomes SUCCESS = 'success' @@ -259,8 +259,8 @@ def __call__(self, source, **kwargs): return self._build(source, **pass_args) else: - print "The source, " + source.name + ", does not exist." - print "Try running update-repos.py first to download the source." + print("The source, %s does not exist." % source.name) + print("Try running update-repos.py first to download the source.") return FAILURE def jenkins_init(self): @@ -311,13 +311,13 @@ def _mac_install_name_fixer(install_manifest): print("Mac HACK: fixing install names for binaries.") + installed_files = [] with open(install_manifest) as fp: - installed_files = fp.readlines() - - installed_files = map(string.strip, installed_files) + for file in fp.readlines(): + installed_files.append(file.strip()) def is_bin(path): - filetype = subprocess.check_output(['file', path]) + filetype = subprocess.check_output(['file', path]).decode('utf-8') result = mac_pkg_helpers.binmatch.search(filetype) return result is not None @@ -330,7 +330,10 @@ def get_deps_to_update(lib_deps, installed): those that we should update the path to """ # We assume that things with @rpath are fine already - abs_paths = filter(lambda p: not p.startswith('@rpath'), lib_deps) + # abs_paths = filter(lambda p: not p.startswith('@rpath'), lib_deps) + abs_paths = [path + for path in lib_deps + if not path.startswith('@rpath')] name = os.path.basename # Now cross-reference them with the other things we've installed -- @@ -359,7 +362,7 @@ def get_rpath(bin): return '@rpath/' + os.path.basename(bin) # update each bin's name and dependencies - for bin, libs in binary_paths.iteritems(): + for (bin, libs) in binary_paths.items(): # We may be doing this unnecessarily in the case of executables # Change where the binary thinks it is new_id = get_rpath(bin) @@ -1020,9 +1023,9 @@ def extract_build_args(args): Intended for use by build-repos and jenkins-build """ kwargs = {} - for k, v in vars(args).iteritems(): - if v is not None: - kwargs[k] = v + for (key, value) in vars(args).items(): + if value is not None: + kwargs[key] = value return kwargs diff --git a/build_tools/common.py b/build_tools/common.py index cb9dca2e..73438c57 100644 --- a/build_tools/common.py +++ b/build_tools/common.py @@ -4,15 +4,16 @@ import os import sys -import artifactory_utils -import builders -import paths -import source -import toolchain -import source_list -import toolchain_list -import utils -import platforms +from jenkins_tools import artifactory_utils as artifactory_utils + +from . import builders +from . import paths +from . import source +from . import toolchain +from . import source_list +from . import toolchain_list +from . import utils +from . import platforms _toolchain_file = os.path.join( os.path.dirname(os.path.abspath(__file__)), @@ -332,7 +333,7 @@ def get_toolchain(parsed_args): if saved_toolchain: def source_match(s): return s.name == source.name - toolchain_sources = filter(source_match, saved_toolchain) + toolchain_sources = list(filter(source_match, saved_toolchain)) if toolchain_sources: assert(len(toolchain_sources) == 1) source = toolchain_sources[0] diff --git a/build_tools/paths.py b/build_tools/paths.py index 1cc6a10a..74499563 100644 --- a/build_tools/paths.py +++ b/build_tools/paths.py @@ -1,7 +1,8 @@ import os -import utils -import platforms + +from . import utils +from . import platforms GITHUB = 'git@github.com:makerbot/' JENKINS = 'git@jenkins.soft.makerbot.net:' @@ -272,7 +273,7 @@ def export_install_dirs(platform, output_file, as_json=False, cross=False): 'UTIL_INSTALL_DIR': 'rootfs/usr/scripts', } - print "writing: " + output_file + print("writing: %s" % output_file) with open(output_file, 'w') as outfile: if as_json: import json diff --git a/build_tools/platforms.py b/build_tools/platforms.py index 5a04804c..a1885e80 100644 --- a/build_tools/platforms.py +++ b/build_tools/platforms.py @@ -7,7 +7,7 @@ import re import subprocess -import utils +from . import utils # The strings below are used directly by jenkins jobs, and changing # them may result in disaster diff --git a/build_tools/source.py b/build_tools/source.py index 6d3b1684..f38686c4 100644 --- a/build_tools/source.py +++ b/build_tools/source.py @@ -10,14 +10,14 @@ import sys import tarfile import time -import types import zipfile -import builders -import paths -import utils -import platforms -import artifactory_utils +from jenkins_tools import artifactory_utils as artifactory_utils + +from . import builders +from . import paths +from . import utils +from . import platforms # for ignoring command output DEVNULL = open(os.devnull, 'wb') @@ -70,7 +70,7 @@ def __init__(self, self.install_manifests = [] self.supported_platforms = [] - if isinstance(supported_platforms, types.StringTypes): + if utils.isstr(supported_platforms): supported_platforms = [supported_platforms] self.supported_platforms.extend(supported_platforms) @@ -210,7 +210,7 @@ def simple_query(): args, cwd=self.dir, shell=self.use_shell, - stderr=stderr) + stderr=stderr).decode('utf-8') if not quiet: print(output) return output @@ -242,7 +242,7 @@ def build(self, *args, **kwargs): """Call the Builder associated with this source, if present""" if self.use_artifacts and not self.handles_artifacts(): # unpack the tarball into the right place. - print ("Unpacking artifacts for " + self.name) + print("Unpacking artifacts for %s" % self.name) # TODO: Update artifactory_utils to explicitly expose this rv = artifactory_utils._unzip_files( self.artifact_path_to_unpack, @@ -541,7 +541,7 @@ def _update(self, tolerant): if rv == "Failure" and not tolerant: return rv elif rv == "Outdated": - print ("Unpacking artifacts for " + self.name) + print("Unpacking artifacts for %s" % self.name) artifact_dir = self.artifact_dir() # Remove everything except the artifact directory diff --git a/build_tools/source_list.py b/build_tools/source_list.py index db9c6962..2af98209 100644 --- a/build_tools/source_list.py +++ b/build_tools/source_list.py @@ -10,11 +10,11 @@ import os import subprocess -import builders -import paths -import platforms -import source -import utils +from . import builders +from . import paths +from . import platforms +from . import source +from . import utils # see below _prototype_hack = ( diff --git a/build_tools/toolchain.py b/build_tools/toolchain.py index 5525c367..14204814 100644 --- a/build_tools/toolchain.py +++ b/build_tools/toolchain.py @@ -1,10 +1,10 @@ import re import collections -import builders -import paths -import utils -import source +from . import builders +from . import paths +from . import utils +from . import source class Toolchain: @@ -79,7 +79,7 @@ def __getattr__(self, name): def __iter__(self): """Iterate over the sources in order""" - return self._sources.itervalues() + return iter(self._sources.values()) def __contains__(self, key): return (key in self._sources) or (key in self.sources()) diff --git a/build_tools/toolchain_list.py b/build_tools/toolchain_list.py index d74213f9..69443f4b 100644 --- a/build_tools/toolchain_list.py +++ b/build_tools/toolchain_list.py @@ -9,10 +9,10 @@ """ import os -import builders -import paths -import toolchain -import source_list +from . import builders +from . import paths +from . import toolchain +from . import source_list # I'm not happy with how much redundancy there diff --git a/build_tools/utils.py b/build_tools/utils.py index 3af2f72d..d337a4f3 100644 --- a/build_tools/utils.py +++ b/build_tools/utils.py @@ -8,7 +8,6 @@ import sys import time import timeit -import types import hashlib from contextlib import contextmanager @@ -61,6 +60,18 @@ def timing_wrapper(*args, **kwargs): return timing_wrapper +# create is string method for both python2 and python3 +try: + # attempt to evaluate basestring. if this fails, then we're in py3 + basestring + + def isstr(s): + return isinstance(s, basestring) +except NameError: + def isstr(s): + return isinstance(s, str) + + @contextmanager def timer(name): """For timing operations @@ -102,7 +113,7 @@ def __init__(self, **kwargs): def __call__(self, parser, namespace, values, option_string=None): # We can get either strings or lists. If we get a string, wrap it. - if isinstance(values, types.StringTypes): + if isstr(values): values = [values] for value in values: diff --git a/check-for-updates.py b/check-for-updates.py index ed400d3f..c2ea43c5 100755 --- a/check-for-updates.py +++ b/check-for-updates.py @@ -27,7 +27,7 @@ def check_for_updates(repo): raise Exception('Could not checkout correct branch') output = repo.query(['git', 'status', '-sb']) - print output + print(output) behind = 'behind' in output if behind: diff --git a/dev-tools/artifactory_clean_repo.py b/dev-tools/artifactory_clean_repo.py index fff33bde..324504df 100644 --- a/dev-tools/artifactory_clean_repo.py +++ b/dev-tools/artifactory_clean_repo.py @@ -24,14 +24,14 @@ all = sorted(all, key=lambda art: art.stat().mtime) old = [i for i in all if (datetime.today()-i.stat().mtime.replace(tzinfo=None)) > age_cutoff] - + for i in all: - print datetime.today()-i.stat().mtime.replace(tzinfo=None); + print(datetime.today()-i.stat().mtime.replace(tzinfo=None)) for i in old: space_made += i.stat().size - print ("deleting: " + str(i)) + print("deleting: %s" % i) # Uncommnet this line to actually delete things #i._accessor.unlink(i) space_made_MB = space_made / 1000000 - print "the " + str(repo) + " repo is using " + str(space_made_MB) + " MB" + print("the %s repo is using %sMB" % (repo, space_made_MB)) diff --git a/dev-tools/artifactory_clean_repo_diagnostic.py b/dev-tools/artifactory_clean_repo_diagnostic.py index b23b070e..09950ccf 100644 --- a/dev-tools/artifactory_clean_repo_diagnostic.py +++ b/dev-tools/artifactory_clean_repo_diagnostic.py @@ -72,13 +72,9 @@ all = sorted(all, key=lambda art: art.stat().mtime) old = [i for i in all if (datetime.today()-i.stat().mtime.replace(tzinfo=None)) > age_cutoff] - - #for i in all: - #print datetime.today()-i.stat().mtime.replace(tzinfo=None); + for i in old: space_made += i.stat().size - #print ("deleting: " + str(i)) - #i._accessor.unlink(i) space_made_MB = space_made / 1000000 - print "the " + str(repo) + " repo is using " + str(space_made_MB) + " MB" + print("the %s repo is using %sMB" % (repo, space_made_MB)) diff --git a/dev-tools/uploadNewArtifacts.py b/dev-tools/uploadNewArtifacts.py index 0ea669f3..2fd4e97c 100755 --- a/dev-tools/uploadNewArtifacts.py +++ b/dev-tools/uploadNewArtifacts.py @@ -91,8 +91,8 @@ def create_artifacts(args): if args.x64: platforms = [p for p in platforms if "64" in p] - print ("Selecting platforms: %s" % platforms) - print ("uploading for version(s): %s" % args.versions) + print("Selecting platforms: %s" % platforms) + print("uploading for version(s): %s" % args.versions) if not args.dry_run: for ver in args.versions: diff --git a/get-dependencies.py b/get-dependencies.py index 40d24944..5db6a636 100755 --- a/get-dependencies.py +++ b/get-dependencies.py @@ -50,6 +50,7 @@ def __init__(self): self.version = None self.build_settings = None - for source in build_tools.get_toolchain(MockParsedArgs()): + for source in build_tools.get_toolchain(MockParsedArgs()).sources(): if isinstance(source, build_tools.source.Repo): + print(source) download_deps(source, args.artifact_dir, args.install_dir) diff --git a/git-hooks/hook_test_rig.py b/git-hooks/hook_test_rig.py index e26cce8a..1cfe27d9 100644 --- a/git-hooks/hook_test_rig.py +++ b/git-hooks/hook_test_rig.py @@ -155,7 +155,7 @@ def test1(): times['checkout'].append(checkout_files(files)) times['checkout all'].append(checkout()) - for k, v in times.iteritems(): + for (k, v) in times.items(): print('{key}:\tmedian={median}\tmean={mean}'.format( key=k, median=sorted(v)[len(v)//2], @@ -176,7 +176,7 @@ def test2(): times['stash'].append(stash()) times['checkout all'].append(checkout()) - for k, v in times.iteritems(): + for (k, v) in times.items(): print('{key}:\tmedian={median}\tmean={mean}'.format( key=k, median=sorted(v)[len(v)//2], diff --git a/jenkins-build.py b/jenkins-build.py index 8dacc983..eb1b8ca0 100755 --- a/jenkins-build.py +++ b/jenkins-build.py @@ -318,7 +318,7 @@ def run(args=[]): # We frequently benefit from knowing the env for a broken build print('Environment (parts may be overridden in code after this point):') print('') - for k, v in sorted(os.environ.iteritems(), key=lambda t: t[0]): + for k, v in os.environ.items(): print('{}: {}'.format(k, v)) print('') @@ -356,8 +356,8 @@ def run(args=[]): subprocess.check_call(' '.join(copy_command), shell=True) if len(repos_failing_tests) > 0: - for r in repos_failing_tests: - print r + " is failing unit tests" + for repo in repos_failing_tests: + print("%s is failing unit tests" % repo) return 0 diff --git a/jenkins_tools/artifactory_utils.py b/jenkins_tools/artifactory_utils.py index 412e9d6e..95757511 100644 --- a/jenkins_tools/artifactory_utils.py +++ b/jenkins_tools/artifactory_utils.py @@ -786,8 +786,8 @@ def all(selector): # get the non-server bits of the path path = selector.path.relative_to(anchor) # make sure you're passing a string to glob, - # it's not built to handle artifactory paths - names = anchor.glob(str(path/selector.name)) + # its not built to handle artifactory paths + names = anchor.glob(str(path) + '/' + selector.name) return selector.filter(names) @@ -816,8 +816,15 @@ def newest(file): def select_multi(selector_list, func): """Convenience wrapper for multiple calls to latest or all""" - result = [func(s) for s in selector_list] - missing_artifacts = [s for r, s in zip(result, selector_list) if r is None] + result = { + s.full_path: func(s) for s in selector_list + } + + missing_artifacts = [] + for (s, r) in result.items(): + if r is None: + missing_artifacts.append(s) + if missing_artifacts: raise RuntimeError("No artifact found matching any of:\n%s" % '\n'.join(str(s) for s in missing_artifacts)) @@ -897,21 +904,21 @@ def download_and_extract_artifacts( If unpacked_cb is passed in it will be invoked after each artifact is unpacked with the artifact as the single argument. """ - for a in artifacts: - print("Looking to download: {}".format(a)) - rv = a.download(artifact_cache) + for selector, artifact in artifacts.items(): + print("Looking to download: {}".format(artifact)) + rv = artifact.download(artifact_cache) if rv == 'Failure': - print("could not download {}".format(a)) + print("could not download {}".format(artifact)) return False - print("Trying to unpack {} to {}".format(a, extract_to)) - rv = a.unzip(artifact_cache, extract_to) + print("Trying to unpack {} to {}".format(artifact, extract_to)) + rv = artifact.unzip(artifact_cache, extract_to) if not rv: - print("could not unpack {}".format(a)) + print("could not unpack {}".format(artifact)) return False if unpacked_cb: - unpacked_cb(a) + unpacked_cb(artifact) return True @@ -1001,7 +1008,8 @@ def checked_convert(artpath): result_list.append(result) # Now any remaining items in the override dict are new and should be added - for artifact_path in override_dict.itervalues(): + # for artifact_path in override_dict.itervalues(): + for artifact_path in override_dict: result = checked_convert(artifact_path) result_list.append(result) diff --git a/jenkins_tools/mac_pkg_helpers.py b/jenkins_tools/mac_pkg_helpers.py index 3c11937d..4e2bc0e7 100644 --- a/jenkins_tools/mac_pkg_helpers.py +++ b/jenkins_tools/mac_pkg_helpers.py @@ -17,7 +17,7 @@ def get_lib_paths(bin_filename, lib_base=''): pattern = re.compile('\s+({}\S+)\s+\(compatibility'.format(lib_base)) for line in otool.stdout.readlines(): - match = pattern.match(line) + match = pattern.match(line.decode('utf-8')) if match is not None: yield match.group(1) @@ -37,11 +37,11 @@ def change_sym(lib_path, sym_dest, bin_filename): return try: - print "Changing install path from" + lib_path + " to " + sym_dest + print("Changing install path from %s to %s" % (lib_path, sym_dest)) subprocess.check_call(['install_name_tool', '-change', lib_path, sym_dest, bin_filename]) except subprocess.CalledProcessError as e: - print 'Error changing install name:' + str(e) + print("Error changing install name: %s" % e) raise @@ -61,7 +61,7 @@ def change_id(bin_dest, bin_filename): bin_dest, bin_filename]) except subprocess.CalledProcessError as e: - print 'Error changing install name:' + str(e) + print("Error changing install name: %s" % e) raise @@ -165,7 +165,7 @@ def fix_lib_paths(fake_root, old_lib_paths, new_lib_base): It also performs a second function where it moves every """ - print('Preparing to fix lib paths') + print("Preparing to fix lib paths") binaries = list(binaries_in_dir(fake_root)) if (str(new_lib_base).startswith('@rpath')): @@ -188,7 +188,7 @@ def fix_lib_paths(fake_root, old_lib_paths, new_lib_base): else: new_lib_path = os.path.join(fake_root, new_lib_base) - print "Fixing binary " + binary + print("Fixing binary %s" % binary) for old_lib_path in old_lib_paths: binaries += fix_binary( @@ -275,10 +275,10 @@ def make_uninstaller(workspace, outfile, resources_dir, scripts_dir, name): outfile]) except OSError as e: - print 'pkgbuild not found: ' + str(e) + print("pkgbuild not found: %s" % e) return 1 except subprocess.CalledProcessError as e: - print 'pkgbuild failed: ' + str(e) + print("pkgbuild failed: %s" % e) return 1 return 0 @@ -301,10 +301,10 @@ def make_app_pkg(app_path, pkg_name): return 1 except OSError as e: - print 'pkgbuild not found: ' + str(e) + print("pkgbuild not found: %s" % e) return 1 except subprocess.CalledProcessError as e: - print 'pkgbuild failed: ' + str(e) + print("pkgbuild failed: %s" % e) return 1 return 0 @@ -327,10 +327,10 @@ def make_root_pkg(root, scripts_dir, identifier, pkg_name): return 1 except OSError as e: - print 'pkgbuild not found: ' + str(e) + print("pkgbuild not found: %s" % e) return 1 except subprocess.CalledProcessError as e: - print 'pkgbuild failed: ' + str(e) + print("pkgbuild failed: %s" % e) return 1 return 0 @@ -351,10 +351,10 @@ def make_installer(package_dir, resources_dir, distribution_xml, outfile): '--resources', resources_dir, outfile]) except OSError as e: - print 'productbuild not found: ' + str(e) + print("productbuild not found: %s" % e) return 1 except subprocess.CalledProcessError as e: - print 'productbuild failed: ' + str(e) + print("productbuild failed: %s" % e) return 1 return 0 diff --git a/status-repos.py b/status-repos.py index 5c6aadf9..4909dc97 100755 --- a/status-repos.py +++ b/status-repos.py @@ -81,7 +81,7 @@ def status_repos(args): jsonblob += sep + ',' + sep + 'version' + sep + ':' jsonblob += sep + source.version + sep + '}' jsonblob += ']' - print jsonblob + print(jsonblob) else: if args.list: def status(source): diff --git a/update-repos.py b/update-repos.py index 7317b1d8..d2448fda 100755 --- a/update-repos.py +++ b/update-repos.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python from __future__ import print_function