Page MenuHome
Differential D5642 Diff 17726 build_files/buildbot/slave_pack.py

Changeset View

Standalone View

View Options

build_files/buildbot/slave_pack.py

Show All 16 Lines
# ##### END GPL LICENSE BLOCK ##### # ##### END GPL LICENSE BLOCK #####
# <pep8 compliant> # <pep8 compliant>
# Runs on buildbot slave, creating a release package using the build # Runs on buildbot slave, creating a release package using the build
# system and zipping it into buildbot_upload.zip. This is then uploaded # system and zipping it into buildbot_upload.zip. This is then uploaded
# to the master in the next buildbot step. # to the master in the next buildbot step.
import buildbot_utils
import os import os
import subprocess
import sys import sys
import zipfile
# get builder name def get_package_name(builder, platform=None):
if len(sys.argv) < 2: info = buildbot_utils.VersionInfo(builder)
sys.stderr.write("Not enough arguments, expecting builder name\n")
sys.exit(1)
builder = sys.argv[1] package_name = 'blender-' + info.version + '-' + info.hash
# Never write branch if it is master. if platform:
branch = sys.argv[2] if (len(sys.argv) >= 3 and sys.argv[2] != 'master') else '' package_name += '-' + platform
if builder.branch != 'master':
blender_dir = os.path.join('..', 'blender.git') package_name = builder.branch + "-" + package_name
build_dir = os.path.join('..', 'build', builder)
install_dir = os.path.join('..', 'install', builder)
buildbot_upload_zip = os.path.abspath(os.path.join(os.path.dirname(install_dir), "buildbot_upload.zip"))
upload_filename = None # Name of the archive to be uploaded
# (this is the name of archive which will appear on the
# download page)
upload_filepath = None # Filepath to be uploaded to the server
# (this folder will be packed)
def parse_header_file(filename, define):
import re
regex = re.compile("^#\s*define\s+%s\s+(.*)" % define)
with open(filename, "r") as file:
for l in file:
match = regex.match(l)
if match:
return match.group(1)
return None
return package_name
# Make sure install directory always exists def create_buildbot_upload_zip(builder, package_filepath, package_filename):
if not os.path.exists(install_dir): import zipfile
os.makedirs(install_dir)
buildbot_upload_zip = os.path.join(builder.upload_dir, "buildbot_upload.zip")
if os.path.exists(buildbot_upload_zip):
os.remove(buildbot_upload_zip)
try:
z = zipfile.ZipFile(buildbot_upload_zip, "w", compression=zipfile.ZIP_STORED)
z.write(package_filepath, arcname=package_filename)
z.close()
except Exception as ex:
sys.stderr.write('Create buildbot_upload.zip failed: ' + str(ex) + '\n')
sys.exit(1)
def create_tar_bz2(src, dest, package_name): def create_tar_bz2(src, dest, package_name):
# One extra to remove leading os.sep when cleaning root for package_root # One extra to remove leading os.sep when cleaning root for package_root
ln = len(src) + 1 ln = len(src) + 1
flist = list() flist = list()
# Create list of tuples containing file and archive name # Create list of tuples containing file and archive name
for root, dirs, files in os.walk(src): for root, dirs, files in os.walk(src):
package_root = os.path.join(package_name, root[ln:]) package_root = os.path.join(package_name, root[ln:])
flist.extend([(os.path.join(root, file), os.path.join(package_root, file)) for file in files]) flist.extend([(os.path.join(root, file), os.path.join(package_root, file)) for file in files])
import tarfile import tarfile
package = tarfile.open(dest, 'w:bz2') package = tarfile.open(dest, 'w:bz2')
for entry in flist: for entry in flist:
package.add(entry[0], entry[1], recursive=False) package.add(entry[0], entry[1], recursive=False)
package.close() package.close()
def cleanup_files(dirpath, extension):
for f in os.listdir(dirpath):
filepath = os.path.join(dirpath, f)
if os.path.isfile(filepath) and f.endswith(extension):
os.remove(filepath)
def find_file(dirpath, extension):
for f in os.listdir(dirpath):
filepath = os.path.join(dirpath, f)
if os.path.isfile(filepath) and f.endswith(extension):
return f
return None
if builder.find('cmake') != -1:
# CMake
if 'win' in builder or 'mac' in builder:
os.chdir(build_dir)
files = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.zip')]
for f in files:
os.remove(f)
retcode = subprocess.call(['cpack', '-G', 'ZIP'])
result_file = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.zip')][0]
# TODO(sergey): Such magic usually happens in SCon's packaging but we don't have it
# in the CMake yet. For until then we do some magic here.
tokens = result_file.split('-')
blender_version = tokens[1].split('.')
blender_full_version = '.'.join(blender_version[0:2])
git_hash = tokens[2].split('.')[1]
platform = builder.split('_')[0]
if platform == 'mac':
# Special exception for OSX
platform = 'OSX-10.9-'
if builder.endswith('x86_64_10_9_cmake'):
platform += 'x86_64'
if builder.endswith('vc2015'):
platform += "-vc14"
builderified_name = 'blender-{}-{}-{}'.format(blender_full_version, git_hash, platform)
# NOTE: Blender 2.7 is already respected by blender_full_version.
if branch != '' and branch != 'blender2.7':
builderified_name = branch + "-" + builderified_name
os.rename(result_file, "{}.zip".format(builderified_name)) def pack_mac(builder):
# create zip file os.chdir(builder.build_dir)
try: cleanup_files(builder.build_dir, '.zip')
if os.path.exists(buildbot_upload_zip):
os.remove(buildbot_upload_zip)
z = zipfile.ZipFile(buildbot_upload_zip, "w", compression=zipfile.ZIP_STORED)
z.write("{}.zip".format(builderified_name))
z.close()
sys.exit(retcode)
except Exception as ex:
sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')
sys.exit(1)
elif builder.startswith('linux_'): package_name = get_package_name(builder, 'OSX-10.9-x86_64')
blender = os.path.join(install_dir, 'blender') package_filename = package_name + '.zip'
buildinfo_h = os.path.join(build_dir, "source", "creator", "buildinfo.h") buildbot_utils.call(['cpack', '-G', 'ZIP'])
blender_h = os.path.join(blender_dir, "source", "blender", "blenkernel", "BKE_blender_version.h") package_filepath = find_file(builder.build_dir, '.zip')
create_buildbot_upload_zip(builder, package_filepath, package_filename)
# Get version information
blender_version = int(parse_header_file(blender_h, 'BLENDER_VERSION'))
blender_version = "%d.%d" % (blender_version // 100, blender_version % 100)
blender_hash = parse_header_file(buildinfo_h, 'BUILD_HASH')[1:-1]
blender_glibc = builder.split('_')[1]
command_prefix = []
bits = 64
blender_arch = 'x86_64'
if blender_glibc == 'glibc224': def pack_win(builder):
if builder.endswith('x86_64_cmake'): os.chdir(builder.build_dir)
chroot_name = 'buildbot_stretch_x86_64' cleanup_files(builder.build_dir, '.zip')
elif builder.endswith('i686_cmake'):
chroot_name = 'buildbot_stretch_i686' package_name = get_package_name(builder, 'win' + str(builder.bits))
bits = 32 package_filename = package_name + '.zip'
blender_arch = 'i686'
command_prefix = ['schroot', '-c', chroot_name, '--'] buildbot_utils.call(['cpack', '-G', 'ZIP'])
elif blender_glibc == 'glibc217': package_filepath = find_file(builder.build_dir, '.zip')
command_prefix = ['scl', 'enable', 'devtoolset-6', '--']
create_buildbot_upload_zip(builder, package_filepath, package_filename)
def pack_linux(builder):
blender_executable = os.path.join(builder.install_dir, 'blender')
info = buildbot_utils.VersionInfo(builder)
blender_glibc = builder.name.split('_')[1]
blender_arch = 'x86_64'
# Strip all unused symbols from the binaries # Strip all unused symbols from the binaries
print("Stripping binaries...") print("Stripping binaries...")
subprocess.call(command_prefix + ['strip', '--strip-all', blender]) buildbot_utils.call(builder.command_prefix + ['strip', '--strip-all', blender_executable])
print("Stripping python...") print("Stripping python...")
py_target = os.path.join(install_dir, blender_version) py_target = os.path.join(builder.install_dir, info.version)
subprocess.call(command_prefix + ['find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';']) buildbot_utils.call(builder.command_prefix + ['find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';'])
# Copy all specific files which are too specific to be copied by # Copy all specific files which are too specific to be copied by
# the CMake rules themselves # the CMake rules themselves
print("Copying extra scripts and libs...") print("Copying extra scripts and libs...")
extra = '/' + os.path.join('home', 'sources', 'release-builder', 'extra') extra = '/' + os.path.join('home', 'sources', 'release-builder', 'extra')
mesalibs = os.path.join(extra, 'mesalibs' + str(bits) + '.tar.bz2') mesalibs = os.path.join(extra, 'mesalibs' + str(builder.bits) + '.tar.bz2')
software_gl = os.path.join(blender_dir, 'release', 'bin', 'blender-softwaregl') software_gl = os.path.join(builder.blender_dir, 'release', 'bin', 'blender-softwaregl')
icons = os.path.join(blender_dir, 'release', 'freedesktop', 'icons') icons = os.path.join(builder.blender_dir, 'release', 'freedesktop', 'icons')
os.system('tar -xpf %s -C %s' % (mesalibs, install_dir)) os.system('tar -xpf %s -C %s' % (mesalibs, builder.install_dir))
os.system('cp %s %s' % (software_gl, install_dir)) os.system('cp %s %s' % (software_gl, builder.install_dir))
os.system('cp -r %s %s' % (icons, install_dir)) os.system('cp -r %s %s' % (icons, builder.install_dir))
os.system('chmod 755 %s' % (os.path.join(install_dir, 'blender-softwaregl'))) os.system('chmod 755 %s' % (os.path.join(builder.install_dir, 'blender-softwaregl')))
# Construct archive name # Construct package name
package_name = 'blender-%s-%s-linux-%s-%s' % (blender_version, platform_name = 'linux-' + blender_glibc + '-' + blender_arch
blender_hash, package_name = get_package_name(builder, platform_name)
blender_glibc, package_filename = package_name + ".tar.bz2"
blender_arch)
# NOTE: Blender 2.7 is already respected by blender_full_version.
if branch != '' and branch != 'blender2.7':
package_name = branch + "-" + package_name
upload_filename = package_name + ".tar.bz2"
print("Creating .tar.bz2 archive") print("Creating .tar.bz2 archive")
upload_filepath = install_dir + '.tar.bz2' package_filepath = builder.install_dir + '.tar.bz2'
create_tar_bz2(install_dir, upload_filepath, package_name) create_tar_bz2(builder.install_dir, package_filepath, package_name)
else:
print("Unknown building system")
sys.exit(1)
if upload_filepath is None:
# clean release directory if it already exists
release_dir = 'release'
if os.path.exists(release_dir):
for f in os.listdir(release_dir):
if os.path.isfile(os.path.join(release_dir, f)):
os.remove(os.path.join(release_dir, f))
# create release package # Create buildbot_upload.zip
try: create_buildbot_upload_zip(builder, package_filepath, package_filename)
subprocess.call(['make', 'package_archive'])
except Exception as ex:
sys.stderr.write('Make package release failed' + str(ex) + '\n')
sys.exit(1)
# find release directory, must exist this time
if not os.path.exists(release_dir):
sys.stderr.write("Failed to find release directory %r.\n" % release_dir)
sys.exit(1)
# find release package
file = None
filepath = None
for f in os.listdir(release_dir):
rf = os.path.join(release_dir, f)
if os.path.isfile(rf) and f.startswith('blender'):
file = f
filepath = rf
if not file: if __name__ == "__main__":
sys.stderr.write("Failed to find release package.\n") builder = buildbot_utils.create_builder_from_arguments()
sys.exit(1)
upload_filename = file # Make sure install directory always exists
upload_filepath = filepath os.makedirs(builder.install_dir, exist_ok=True)
# create zip file if builder.platform == 'mac':
try: pack_mac(builder)
upload_zip = os.path.join(buildbot_upload_zip) elif builder.platform == 'win':
if os.path.exists(upload_zip): pack_win(builder)
os.remove(upload_zip) elif builder.platform == 'linux':
z = zipfile.ZipFile(upload_zip, "w", compression=zipfile.ZIP_STORED) pack_linux(builder)
z.write(upload_filepath, arcname=upload_filename)
z.close()
except Exception as ex:
sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')
sys.exit(1)