dom
/
telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

build.py: Make build script work on both Python2.x and Python3.x 

While at it also add a missing dependency on lsof required by the
netstat plugin.

closes #512
This commit is contained in:
Hannu Valtonen 2016-01-12 23:10:36 +02:00 committed by Cameron Sparr
parent 3cc1fecb53
commit 7531e218c1
2 changed files with 123 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -2,6 +2,7 @@
### Features ### Features
- [#509](https://github.com/influxdb/telegraf/pull/509): Flatten JSON arrays with indices. Thanks @psilva261! - [#509](https://github.com/influxdb/telegraf/pull/509): Flatten JSON arrays with indices. Thanks @psilva261!
- [#512](https://github.com/influxdata/telegraf/pull/512): Python 3 build script, add lsof dep to package. Thanks @Ormod!
### Bugfixes ### Bugfixes
- [#506](https://github.com/influxdb/telegraf/pull/506): Ping input doesn't return response time metric when timeout. Thanks @titilambert! - [#506](https://github.com/influxdb/telegraf/pull/506): Ping input doesn't return response time metric when timeout. Thanks @titilambert!

240
build.py
View File

@ -92,31 +92,32 @@ def run(command, allow_failure=False, shell=False):
out = subprocess.check_output(command, stderr=subprocess.STDOUT, shell=shell) out = subprocess.check_output(command, stderr=subprocess.STDOUT, shell=shell)
else: else:
out = subprocess.check_output(command.split(), stderr=subprocess.STDOUT) out = subprocess.check_output(command.split(), stderr=subprocess.STDOUT)
out = out.decode("utf8")
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
print "" print("")
print "" print("")
print "Executed command failed!" print("Executed command failed!")
print "-- Command run was: {}".format(command) print("-- Command run was: {}".format(command))
print "-- Failure was: {}".format(e.output) print("-- Failure was: {}".format(e.output))
if allow_failure: if allow_failure:
print "Continuing..." print("Continuing...")
return None return None
else: else:
print "" print("")
print "Stopping." print("Stopping.")
sys.exit(1) sys.exit(1)
except OSError as e: except OSError as e:
print "" print("")
print "" print("")
print "Invalid command!" print("Invalid command!")
print "-- Command run was: {}".format(command) print("-- Command run was: {}".format(command))
print "-- Failure was: {}".format(e) print("-- Failure was: {}".format(e))
if allow_failure: if allow_failure:
print "Continuing..." print("Continuing...")
return out return out
else: else:
print "" print("")
print "Stopping." print("Stopping.")
sys.exit(1) sys.exit(1)
else: else:
return out return out
@ -173,42 +174,42 @@ def check_path_for(b):
return full_path return full_path
def check_environ(build_dir = None): def check_environ(build_dir = None):
print "\nChecking environment:" print("\nChecking environment:")
for v in [ "GOPATH", "GOBIN", "GOROOT" ]: for v in [ "GOPATH", "GOBIN", "GOROOT" ]:
print "\t- {} -> {}".format(v, os.environ.get(v)) print("\t- {} -> {}".format(v, os.environ.get(v)))
cwd = os.getcwd() cwd = os.getcwd()
if build_dir == None and os.environ.get("GOPATH") and os.environ.get("GOPATH") not in cwd: if build_dir == None and os.environ.get("GOPATH") and os.environ.get("GOPATH") not in cwd:
print "\n!! WARNING: Your current directory is not under your GOPATH. This may lead to build failures." print("\n!! WARNING: Your current directory is not under your GOPATH. This may lead to build failures.")
def check_prereqs(): def check_prereqs():
print "\nChecking for dependencies:" print("\nChecking for dependencies:")
for req in prereqs: for req in prereqs:
print "\t- {} ->".format(req), print("\t- {} ->".format(req),)
path = check_path_for(req) path = check_path_for(req)
if path: if path:
print "{}".format(path) print("{}".format(path))
else: else:
print "?" print("?")
for req in optional_prereqs: for req in optional_prereqs:
print "\t- {} (optional) ->".format(req), print("\t- {} (optional) ->".format(req))
path = check_path_for(req) path = check_path_for(req)
if path: if path:
print "{}".format(path) print("{}".format(path))
else: else:
print "?" print("?")
print "" print("")
def upload_packages(packages, nightly=False): def upload_packages(packages, nightly=False):
print "Uploading packages to S3..." print("Uploading packages to S3...")
print "" print("")
c = boto.connect_s3() c = boto.connect_s3()
# TODO(rossmcdonald) - Set to different S3 bucket for release vs nightly # TODO(rossmcdonald) - Set to different S3 bucket for release vs nightly
bucket = c.get_bucket('get.influxdb.org') bucket = c.get_bucket('get.influxdb.org')
for p in packages: for p in packages:
name = os.path.join('telegraf', os.path.basename(p)) name = os.path.join('telegraf', os.path.basename(p))
if bucket.get_key(name) is None or nightly: if bucket.get_key(name) is None or nightly:
print "\t - Uploading {}...".format(name), print("\t - Uploading {}...".format(name))
k = Key(bucket) k = Key(bucket)
k.key = name k.key = name
if nightly: if nightly:
@ -216,41 +217,41 @@ def upload_packages(packages, nightly=False):
else: else:
n = k.set_contents_from_filename(p, replace=False) n = k.set_contents_from_filename(p, replace=False)
k.make_public() k.make_public()
print "[ DONE ]" print("[ DONE ]")
else: else:
print "\t - Not uploading {}, already exists.".format(p) print("\t - Not uploading {}, already exists.".format(p))
print "" print("")
def run_tests(race, parallel, timeout, no_vet): def run_tests(race, parallel, timeout, no_vet):
get_command = "go get -d -t ./..." get_command = "go get -d -t ./..."
print "Retrieving Go dependencies...", print("Retrieving Go dependencies...")
sys.stdout.flush() sys.stdout.flush()
run(get_command) run(get_command)
print "done." print("done.")
print "Running tests:" print("Running tests:")
print "\tRace: ", race print("\tRace: ", race)
if parallel is not None: if parallel is not None:
print "\tParallel:", parallel print("\tParallel:", parallel)
if timeout is not None: if timeout is not None:
print "\tTimeout:", timeout print("\tTimeout:", timeout)
sys.stdout.flush() sys.stdout.flush()
p = subprocess.Popen(["go", "fmt", "./..."], stdout=subprocess.PIPE, stderr=subprocess.PIPE) p = subprocess.Popen(["go", "fmt", "./..."], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate() out, err = p.communicate()
if len(out) > 0 or len(err) > 0: if len(out) > 0 or len(err) > 0:
print "Code not formatted. Please use 'go fmt ./...' to fix formatting errors." print("Code not formatted. Please use 'go fmt ./...' to fix formatting errors.")
print out print(out)
print err print(err)
return False return False
if not no_vet: if not no_vet:
p = subprocess.Popen(["go", "tool", "vet", "-composites=false", "./"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) p = subprocess.Popen(["go", "tool", "vet", "-composites=false", "./"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate() out, err = p.communicate()
if len(out) > 0 or len(err) > 0: if len(out) > 0 or len(err) > 0:
print "Go vet failed. Please run 'go vet ./...' and fix any errors." print("Go vet failed. Please run 'go vet ./...' and fix any errors.")
print out print(out)
print err print(err)
return False return False
else: else:
print "Skipping go vet ..." print("Skipping go vet ...")
sys.stdout.flush() sys.stdout.flush()
test_command = "go test -v" test_command = "go test -v"
if race: if race:
@ -262,10 +263,10 @@ def run_tests(race, parallel, timeout, no_vet):
test_command += " ./..." test_command += " ./..."
code = os.system(test_command) code = os.system(test_command)
if code != 0: if code != 0:
print "Tests Failed" print("Tests Failed")
return False return False
else: else:
print "Tests Passed" print("Tests Passed")
return True return True
def build(version=None, def build(version=None,
@ -279,26 +280,26 @@ def build(version=None,
clean=False, clean=False,
outdir=".", outdir=".",
goarm_version="6"): goarm_version="6"):
print "-------------------------" print("-------------------------")
print "" print("")
print "Build plan:" print("Build plan:")
print "\t- version: {}".format(version) print("\t- version: {}".format(version))
if rc: if rc:
print "\t- release candidate: {}".format(rc) print("\t- release candidate: {}".format(rc))
print "\t- commit: {}".format(commit) print("\t- commit: {}".format(commit))
print "\t- branch: {}".format(branch) print("\t- branch: {}".format(branch))
print "\t- platform: {}".format(platform) print("\t- platform: {}".format(platform))
print "\t- arch: {}".format(arch) print("\t- arch: {}".format(arch))
if arch == 'arm' and goarm_version: if arch == 'arm' and goarm_version:
print "\t- ARM version: {}".format(goarm_version) print("\t- ARM version: {}".format(goarm_version))
print "\t- nightly? {}".format(str(nightly).lower()) print("\t- nightly? {}".format(str(nightly).lower()))
print "\t- race enabled? {}".format(str(race).lower()) print("\t- race enabled? {}".format(str(race).lower()))
print "" print("")
if not os.path.exists(outdir): if not os.path.exists(outdir):
os.makedirs(outdir) os.makedirs(outdir)
elif clean and outdir != '/': elif clean and outdir != '/':
print "Cleaning build directory..." print("Cleaning build directory...")
shutil.rmtree(outdir) shutil.rmtree(outdir)
os.makedirs(outdir) os.makedirs(outdir)
@ -306,14 +307,14 @@ def build(version=None,
# If a release candidate, update the version information accordingly # If a release candidate, update the version information accordingly
version = "{}rc{}".format(version, rc) version = "{}rc{}".format(version, rc)
print "Starting build..." print("Starting build...")
for b, c in targets.iteritems(): for b, c in targets.items():
print "\t- Building '{}'...".format(os.path.join(outdir, b)), print("\t- Building '{}'...".format(os.path.join(outdir, b)),)
build_command = "" build_command = ""
build_command += "GOOS={} GOARCH={} ".format(platform, arch) build_command += "GOOS={} GOARCH={} ".format(platform, arch)
if arch == "arm" and goarm_version: if arch == "arm" and goarm_version:
if goarm_version not in ["5", "6", "7", "arm64"]: if goarm_version not in ["5", "6", "7", "arm64"]:
print "!! Invalid ARM build version: {}".format(goarm_version) print("!! Invalid ARM build version: {}".format(goarm_version))
build_command += "GOARM={} ".format(goarm_version) build_command += "GOARM={} ".format(goarm_version)
build_command += "go build -o {} ".format(os.path.join(outdir, b)) build_command += "go build -o {} ".format(os.path.join(outdir, b))
if race: if race:
@ -331,20 +332,20 @@ def build(version=None,
build_command += "-X main.Commit={}\" ".format(get_current_commit()) build_command += "-X main.Commit={}\" ".format(get_current_commit())
build_command += c build_command += c
run(build_command, shell=True) run(build_command, shell=True)
print "[ DONE ]" print("[ DONE ]")
print "" print("")
def create_dir(path): def create_dir(path):
try: try:
os.makedirs(path) os.makedirs(path)
except OSError as e: except OSError as e:
print e print(e)
def rename_file(fr, to): def rename_file(fr, to):
try: try:
os.rename(fr, to) os.rename(fr, to)
except OSError as e: except OSError as e:
print e print(e)
# Return the original filename # Return the original filename
return fr return fr
else: else:
@ -355,27 +356,27 @@ def copy_file(fr, to):
try: try:
shutil.copy(fr, to) shutil.copy(fr, to)
except OSError as e: except OSError as e:
print e print(e)
def create_package_fs(build_root): def create_package_fs(build_root):
print "\t- Creating a filesystem hierarchy from directory: {}".format(build_root) print("\t- Creating a filesystem hierarchy from directory: {}".format(build_root))
# Using [1:] for the path names due to them being absolute # Using [1:] for the path names due to them being absolute
# (will overwrite previous paths, per 'os.path.join' documentation) # (will overwrite previous paths, per 'os.path.join' documentation)
dirs = [ INSTALL_ROOT_DIR[1:], LOG_DIR[1:], SCRIPT_DIR[1:], CONFIG_DIR[1:], LOGROTATE_DIR[1:] ] dirs = [ INSTALL_ROOT_DIR[1:], LOG_DIR[1:], SCRIPT_DIR[1:], CONFIG_DIR[1:], LOGROTATE_DIR[1:] ]
for d in dirs: for d in dirs:
create_dir(os.path.join(build_root, d)) create_dir(os.path.join(build_root, d))
os.chmod(os.path.join(build_root, d), 0755) os.chmod(os.path.join(build_root, d), 0o755)
def package_scripts(build_root): def package_scripts(build_root):
print "\t- Copying scripts and sample configuration to build directory" print("\t- Copying scripts and sample configuration to build directory")
shutil.copyfile(INIT_SCRIPT, os.path.join(build_root, SCRIPT_DIR[1:], INIT_SCRIPT.split('/')[1])) shutil.copyfile(INIT_SCRIPT, os.path.join(build_root, SCRIPT_DIR[1:], INIT_SCRIPT.split('/')[1]))
os.chmod(os.path.join(build_root, SCRIPT_DIR[1:], INIT_SCRIPT.split('/')[1]), 0644) os.chmod(os.path.join(build_root, SCRIPT_DIR[1:], INIT_SCRIPT.split('/')[1]), 0o644)
shutil.copyfile(SYSTEMD_SCRIPT, os.path.join(build_root, SCRIPT_DIR[1:], SYSTEMD_SCRIPT.split('/')[1])) shutil.copyfile(SYSTEMD_SCRIPT, os.path.join(build_root, SCRIPT_DIR[1:], SYSTEMD_SCRIPT.split('/')[1]))
os.chmod(os.path.join(build_root, SCRIPT_DIR[1:], SYSTEMD_SCRIPT.split('/')[1]), 0644) os.chmod(os.path.join(build_root, SCRIPT_DIR[1:], SYSTEMD_SCRIPT.split('/')[1]), 0o644)
shutil.copyfile(LOGROTATE_SCRIPT, os.path.join(build_root, LOGROTATE_DIR[1:], "telegraf")) shutil.copyfile(LOGROTATE_SCRIPT, os.path.join(build_root, LOGROTATE_DIR[1:], "telegraf"))
os.chmod(os.path.join(build_root, LOGROTATE_DIR[1:], "telegraf"), 0644) os.chmod(os.path.join(build_root, LOGROTATE_DIR[1:], "telegraf"), 0o644)
shutil.copyfile(DEFAULT_CONFIG, os.path.join(build_root, CONFIG_DIR[1:], "telegraf.conf")) shutil.copyfile(DEFAULT_CONFIG, os.path.join(build_root, CONFIG_DIR[1:], "telegraf.conf"))
os.chmod(os.path.join(build_root, CONFIG_DIR[1:], "telegraf.conf"), 0644) os.chmod(os.path.join(build_root, CONFIG_DIR[1:], "telegraf.conf"), 0o644)
def go_get(update=False): def go_get(update=False):
get_command = None get_command = None
@ -383,24 +384,27 @@ def go_get(update=False):
get_command = "go get -u -f -d ./..." get_command = "go get -u -f -d ./..."
else: else:
get_command = "go get -d ./..." get_command = "go get -d ./..."
print "Retrieving Go dependencies...", print("Retrieving Go dependencies...")
run(get_command) run(get_command)
print "done.\n" print("done.\n")
def generate_md5_from_file(path): def generate_md5_from_file(path):
m = hashlib.md5() m = hashlib.md5()
with open(path, 'rb') as f: with open(path, 'rb') as f:
for chunk in iter(lambda: f.read(4096), b""): while True:
m.update(chunk) data = f.read(4096)
if not data:
break
m.update(data)
return m.hexdigest() return m.hexdigest()
def build_packages(build_output, version, nightly=False, rc=None, iteration=1): def build_packages(build_output, version, nightly=False, rc=None, iteration=1):
outfiles = [] outfiles = []
tmp_build_dir = create_temp_dir() tmp_build_dir = create_temp_dir()
try: try:
print "-------------------------" print("-------------------------")
print "" print("")
print "Packaging..." print("Packaging...")
for p in build_output: for p in build_output:
# Create top-level folder displaying which platform (linux, etc) # Create top-level folder displaying which platform (linux, etc)
create_dir(os.path.join(tmp_build_dir, p)) create_dir(os.path.join(tmp_build_dir, p))
@ -419,11 +423,11 @@ def build_packages(build_output, version, nightly=False, rc=None, iteration=1):
b = b + '.exe' b = b + '.exe'
fr = os.path.join(current_location, b) fr = os.path.join(current_location, b)
to = os.path.join(build_root, INSTALL_ROOT_DIR[1:], b) to = os.path.join(build_root, INSTALL_ROOT_DIR[1:], b)
print "\t- [{}][{}] - Moving from '{}' to '{}'".format(p, a, fr, to) print("\t- [{}][{}] - Moving from '{}' to '{}'".format(p, a, fr, to))
copy_file(fr, to) copy_file(fr, to)
# Package the directory structure # Package the directory structure
for package_type in supported_packages[p]: for package_type in supported_packages[p]:
print "\t- Packaging directory '{}' as '{}'...".format(build_root, package_type), print("\t- Packaging directory '{}' as '{}'...".format(build_root, package_type))
name = "telegraf" name = "telegraf"
package_version = version package_version = version
package_iteration = iteration package_iteration = iteration
@ -448,52 +452,53 @@ def build_packages(build_output, version, nightly=False, rc=None, iteration=1):
current_location) current_location)
if package_type == "rpm": if package_type == "rpm":
fpm_command += "--depends coreutils " fpm_command += "--depends coreutils "
fpm_command += "--depends lsof"
out = run(fpm_command, shell=True) out = run(fpm_command, shell=True)
matches = re.search(':path=>"(.*)"', out) matches = re.search(':path=>"(.*)"', out)
outfile = None outfile = None
if matches is not None: if matches is not None:
outfile = matches.groups()[0] outfile = matches.groups()[0]
if outfile is None: if outfile is None:
print "[ COULD NOT DETERMINE OUTPUT ]" print("[ COULD NOT DETERMINE OUTPUT ]")
else: else:
# Strip nightly version (the unix epoch) from filename # Strip nightly version (the unix epoch) from filename
if nightly and package_type in ['deb', 'rpm']: if nightly and package_type in ['deb', 'rpm']:
outfile = rename_file(outfile, outfile.replace("{}-{}".format(version, iteration), "nightly")) outfile = rename_file(outfile, outfile.replace("{}-{}".format(version, iteration), "nightly"))
outfiles.append(os.path.join(os.getcwd(), outfile)) outfiles.append(os.path.join(os.getcwd(), outfile))
print "[ DONE ]" print("[ DONE ]")
# Display MD5 hash for generated package # Display MD5 hash for generated package
print "\t\tMD5 = {}".format(generate_md5_from_file(outfile)) print("\t\tMD5 = {}".format(generate_md5_from_file(outfile)))
print "" print("")
return outfiles return outfiles
finally: finally:
# Cleanup # Cleanup
shutil.rmtree(tmp_build_dir) shutil.rmtree(tmp_build_dir)
def print_usage(): def print_usage():
print "Usage: ./build.py [options]" print("Usage: ./build.py [options]")
print "" print("")
print "Options:" print("Options:")
print "\t --outdir=<path> \n\t\t- Send build output to a specified path. Defaults to ./build." print("\t --outdir=<path> \n\t\t- Send build output to a specified path. Defaults to ./build.")
print "\t --arch=<arch> \n\t\t- Build for specified architecture. Acceptable values: x86_64|amd64, 386, arm, or all" print("\t --arch=<arch> \n\t\t- Build for specified architecture. Acceptable values: x86_64|amd64, 386, arm, or all")
print "\t --goarm=<arm version> \n\t\t- Build for specified ARM version (when building for ARM). Default value is: 6" print("\t --goarm=<arm version> \n\t\t- Build for specified ARM version (when building for ARM). Default value is: 6")
print "\t --platform=<platform> \n\t\t- Build for specified platform. Acceptable values: linux, windows, darwin, or all" print("\t --platform=<platform> \n\t\t- Build for specified platform. Acceptable values: linux, windows, darwin, or all")
print "\t --version=<version> \n\t\t- Version information to apply to build metadata. If not specified, will be pulled from repo tag." print("\t --version=<version> \n\t\t- Version information to apply to build metadata. If not specified, will be pulled from repo tag.")
print "\t --commit=<commit> \n\t\t- Use specific commit for build (currently a NOOP)." print("\t --commit=<commit> \n\t\t- Use specific commit for build (currently a NOOP).")
print "\t --branch=<branch> \n\t\t- Build from a specific branch (currently a NOOP)." print("\t --branch=<branch> \n\t\t- Build from a specific branch (currently a NOOP).")
print "\t --rc=<rc number> \n\t\t- Whether or not the build is a release candidate (affects version information)." print("\t --rc=<rc number> \n\t\t- Whether or not the build is a release candidate (affects version information).")
print "\t --iteration=<iteration number> \n\t\t- The iteration to display on the package output (defaults to 0 for RC's, and 1 otherwise)." print("\t --iteration=<iteration number> \n\t\t- The iteration to display on the package output (defaults to 0 for RC's, and 1 otherwise).")
print "\t --race \n\t\t- Whether the produced build should have race detection enabled." print("\t --race \n\t\t- Whether the produced build should have race detection enabled.")
print "\t --package \n\t\t- Whether the produced builds should be packaged for the target platform(s)." print("\t --package \n\t\t- Whether the produced builds should be packaged for the target platform(s).")
print "\t --nightly \n\t\t- Whether the produced build is a nightly (affects version information)." print("\t --nightly \n\t\t- Whether the produced build is a nightly (affects version information).")
print "\t --update \n\t\t- Whether dependencies should be updated prior to building." print("\t --update \n\t\t- Whether dependencies should be updated prior to building.")
print "\t --test \n\t\t- Run Go tests. Will not produce a build." print("\t --test \n\t\t- Run Go tests. Will not produce a build.")
print "\t --parallel \n\t\t- Run Go tests in parallel up to the count specified." print("\t --parallel \n\t\t- Run Go tests in parallel up to the count specified.")
print "\t --timeout \n\t\t- Timeout for Go tests. Defaults to 480s." print("\t --timeout \n\t\t- Timeout for Go tests. Defaults to 480s.")
print "\t --clean \n\t\t- Clean the build output directory prior to creating build." print("\t --clean \n\t\t- Clean the build output directory prior to creating build.")
print "" print("")
def print_package_summary(packages): def print_package_summary(packages):
print packages print(packages)
def main(): def main():
# Command-line arguments # Command-line arguments
@ -577,13 +582,13 @@ def main():
print_usage() print_usage()
return 0 return 0
else: else:
print "!! Unknown argument: {}".format(arg) print("!! Unknown argument: {}".format(arg))
print_usage() print_usage()
return 1 return 1
if nightly: if nightly:
if rc: if rc:
print "!! Cannot be both nightly and a release candidate! Stopping." print("!! Cannot be both nightly and a release candidate! Stopping.")
return 1 return 1
# In order to support nightly builds on the repository, we are adding the epoch timestamp # In order to support nightly builds on the repository, we are adding the epoch timestamp
# to the version so that version numbers are always greater than the previous nightly. # to the version so that version numbers are always greater than the previous nightly.
@ -622,7 +627,7 @@ def main():
platforms = [] platforms = []
single_build = True single_build = True
if target_platform == 'all': if target_platform == 'all':
platforms = supported_builds.keys() platforms = list(supported_builds.keys())
single_build = False single_build = False
else: else:
platforms = [target_platform] platforms = [target_platform]
@ -655,7 +660,7 @@ def main():
# Build packages # Build packages
if package: if package:
if not check_path_for("fpm"): if not check_path_for("fpm"):
print "!! Cannot package without command 'fpm'. Stopping." print("!! Cannot package without command 'fpm'. Stopping.")
return 1 return 1
packages = build_packages(build_output, version, nightly=nightly, rc=rc, iteration=iteration) packages = build_packages(build_output, version, nightly=nightly, rc=rc, iteration=iteration)
# TODO(rossmcdonald): Add nice output for print_package_summary() # TODO(rossmcdonald): Add nice output for print_package_summary()
@ -667,4 +672,3 @@ def main():
if __name__ == '__main__': if __name__ == '__main__':
sys.exit(main()) sys.exit(main())