From 1847ce3f3d304e27eb92e8a68e60308defe7ddbc Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Mon, 22 Feb 2016 15:16:46 -0700 Subject: [PATCH] Experimental windows build process changes --- Godeps_windows | 8 +-- etc/telegraf_windows.conf | 120 ++++++++++++++++++++++++++++++++++++++ scripts/build.py | 101 +++++++++++++++++++------------- scripts/circle-test.sh | 1 + 4 files changed, 185 insertions(+), 45 deletions(-) create mode 100644 etc/telegraf_windows.conf diff --git a/Godeps_windows b/Godeps_windows index 034fb4fec..dd46184ec 100644 --- a/Godeps_windows +++ b/Godeps_windows @@ -1,4 +1,4 @@ -git.eclipse.org/gitroot/paho/org.eclipse.paho.mqtt.golang.git dbd8d5c40a582eb9adacde36b47932b3a3ad0034 +git.eclipse.org/gitroot/paho/org.eclipse.paho.mqtt.golang.git 617c801af238c3af2d9e72c5d4a0f02edad03ce5 github.com/Shopify/sarama d37c73f2b2bce85f7fa16b6a550d26c5372892ef github.com/Sirupsen/logrus f7f79f729e0fbe2fcc061db48a9ba0263f588252 github.com/StackExchange/wmi f3e2bae1e0cb5aef83e319133eabfee30013a4a5 @@ -21,18 +21,18 @@ github.com/gorilla/context 1c83b3eabd45b6d76072b66b746c20815fb2872d github.com/gorilla/mux 26a6070f849969ba72b72256e9f14cf519751690 github.com/hailocab/go-hostpool e80d13ce29ede4452c43dea11e79b9bc8a15b478 github.com/influxdata/config bae7cb98197d842374d3b8403905924094930f24 -github.com/influxdata/influxdb a9552fdd91361819a792f337e5d9998859732a67 -github.com/influxdb/influxdb a9552fdd91361819a792f337e5d9998859732a67 +github.com/influxdata/influxdb ef571fc104dc24b77cd3710c156cd95e5cfd7aa5 github.com/jmespath/go-jmespath c01cf91b011868172fdcd9f41838e80c9d716264 github.com/klauspost/crc32 999f3125931f6557b991b2f8472172bdfa578d38 github.com/lib/pq 8ad2b298cadd691a77015666a5372eae5dbfac8f github.com/lxn/win 9a7734ea4db26bc593d52f6a8a957afdad39c5c1 github.com/matttproud/golang_protobuf_extensions d0c3fe89de86839aecf2e0579c40ba3bb336a453 +github.com/miekg/dns e0d84d97e59bcb6561eae269c4e94d25b66822cb github.com/mreiferson/go-snappystream 028eae7ab5c4c9e2d1cb4c4ca1e53259bbe7e504 github.com/naoina/go-stringutil 6b638e95a32d0c1131db0e7fe83775cbea4a0d0b github.com/naoina/toml 751171607256bb66e64c9f0220c00662420c38e9 +github.com/nats-io/nats 6a83f1a633cfbfd90aa648ac99fb38c06a8b40df github.com/nsqio/go-nsq 2118015c120962edc5d03325c680daf3163a8b5f -github.com/pborman/uuid dee7705ef7b324f27ceb85a121c61f2c2e8ce988 github.com/pmezard/go-difflib 792786c7400a136282c1664665ae0a8db921c6c2 github.com/prometheus/client_golang 67994f177195311c3ea3d4407ed0175e34a4256f github.com/prometheus/client_model fa8ad6fec33561be4280a8f0514318c79d7f6cb6 diff --git a/etc/telegraf_windows.conf b/etc/telegraf_windows.conf new file mode 100644 index 000000000..df35261d7 --- /dev/null +++ b/etc/telegraf_windows.conf @@ -0,0 +1,120 @@ +# Telegraf configuration + +# Telegraf is entirely plugin driven. All metrics are gathered from the +# declared inputs, and sent to the declared outputs. + +# Plugins must be declared in here to be active. +# To deactivate a plugin, comment out the name and any variables. + +# Use 'telegraf -config telegraf.conf -test' to see what metrics a config +# file would generate. + +# Global tags can be specified here in key="value" format. +[global_tags] + # dc = "us-east-1" # will tag all metrics with dc=us-east-1 + # rack = "1a" + +# Configuration for telegraf agent +[agent] + ## Default data collection interval for all inputs + interval = "10s" + ## Rounds collection interval to 'interval' + ## ie, if interval="10s" then always collect on :00, :10, :20, etc. + round_interval = true + + ## Telegraf will cache metric_buffer_limit metrics for each output, and will + ## flush this buffer on a successful write. + metric_buffer_limit = 10000 + ## Flush the buffer whenever full, regardless of flush_interval. + flush_buffer_when_full = true + + ## Collection jitter is used to jitter the collection by a random amount. + ## Each plugin will sleep for a random time within jitter before collecting. + ## This can be used to avoid many plugins querying things like sysfs at the + ## same time, which can have a measurable effect on the system. + collection_jitter = "0s" + + ## Default flushing interval for all outputs. You shouldn't set this below + ## interval. Maximum flush_interval will be flush_interval + flush_jitter + flush_interval = "10s" + ## Jitter the flush interval by a random amount. This is primarily to avoid + ## large write spikes for users running a large number of telegraf instances. + ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s + flush_jitter = "0s" + + ## Run telegraf in debug mode + debug = false + ## Run telegraf in quiet mode + quiet = false + ## Override default hostname, if empty use os.Hostname() + hostname = "" + + +############################################################################### +# OUTPUTS # +############################################################################### + +# Configuration for influxdb server to send metrics to +[[outputs.influxdb]] + # The full HTTP or UDP endpoint URL for your InfluxDB instance. + # Multiple urls can be specified but it is assumed that they are part of the same + # cluster, this means that only ONE of the urls will be written to each interval. + # urls = ["udp://localhost:8089"] # UDP endpoint example + urls = ["http://localhost:8086"] # required + # The target database for metrics (telegraf will create it if not exists) + database = "telegraf" # required + # Precision of writes, valid values are "ns", "us" (or "µs"), "ms", "s", "m", "h". + # note: using second precision greatly helps InfluxDB compression + precision = "s" + + ## Write timeout (for the InfluxDB client), formatted as a string. + ## If not provided, will default to 5s. 0s means no timeout (not recommended). + timeout = "5s" + # username = "telegraf" + # password = "metricsmetricsmetricsmetrics" + # Set the user agent for HTTP POSTs (can be useful for log differentiation) + # user_agent = "telegraf" + # Set UDP payload size, defaults to InfluxDB UDP Client default (512 bytes) + # udp_payload = 512 + + +############################################################################### +# INPUTS # +############################################################################### + +# Read metrics about cpu usage +[[inputs.cpu]] + # Whether to report per-cpu stats or not + percpu = true + # Whether to report total system cpu stats or not + totalcpu = true + # Comment this line if you want the raw CPU time metrics + fielddrop = ["time_*"] + +# Read metrics about disk usage by mount point +[[inputs.disk]] + # By default, telegraf gather stats for all mountpoints. + # Setting mountpoints will restrict the stats to the specified mountpoints. + # mount_points=["/"] + + # Ignore some mountpoints by filesystem type. For example (dev)tmpfs (usually + # present on /run, /var/run, /dev/shm or /dev). + ignore_fs = ["tmpfs", "devtmpfs"] + +# Read metrics about disk IO by device +[[inputs.diskio]] + # By default, telegraf will gather stats for all devices including + # disk partitions. + # Setting devices will restrict the stats to the specified devices. + # devices = ["sda", "sdb"] + # Uncomment the following line if you do not need disk serial numbers. + # skip_serial_number = true + +# Read metrics about memory usage +[[inputs.mem]] + # no configuration + +# Read metrics about swap memory usage +[[inputs.swap]] + # no configuration + diff --git a/scripts/build.py b/scripts/build.py index 9aca120cd..1465e36f3 100755 --- a/scripts/build.py +++ b/scripts/build.py @@ -30,6 +30,7 @@ INIT_SCRIPT = "scripts/init.sh" SYSTEMD_SCRIPT = "scripts/telegraf.service" LOGROTATE_SCRIPT = "etc/logrotate.d/telegraf" DEFAULT_CONFIG = "etc/telegraf.conf" +DEFAULT_WINDOWS_CONFIG = "etc/telegraf_windows.conf" POSTINST_SCRIPT = "scripts/post-install.sh" PREINST_SCRIPT = "scripts/pre-install.sh" @@ -76,7 +77,7 @@ supported_builds = { supported_packages = { "darwin": [ "tar", "zip" ], "linux": [ "deb", "rpm", "tar", "zip" ], - "windows": [ "tar", "zip" ], + "windows": [ "zip" ], } supported_tags = { # "linux": { @@ -351,20 +352,25 @@ def create_package_fs(build_root): create_dir(os.path.join(build_root, d)) os.chmod(os.path.join(build_root, d), 0o755) -def package_scripts(build_root): +def package_scripts(build_root, windows=False): print("\t- Copying scripts and sample configuration to build directory") - shutil.copyfile(INIT_SCRIPT, os.path.join(build_root, SCRIPT_DIR[1:], INIT_SCRIPT.split('/')[1])) - os.chmod(os.path.join(build_root, SCRIPT_DIR[1:], INIT_SCRIPT.split('/')[1]), 0o644) - shutil.copyfile(SYSTEMD_SCRIPT, os.path.join(build_root, SCRIPT_DIR[1:], SYSTEMD_SCRIPT.split('/')[1])) - os.chmod(os.path.join(build_root, SCRIPT_DIR[1:], SYSTEMD_SCRIPT.split('/')[1]), 0o644) - shutil.copyfile(LOGROTATE_SCRIPT, os.path.join(build_root, LOGROTATE_DIR[1:], "telegraf")) - os.chmod(os.path.join(build_root, LOGROTATE_DIR[1:], "telegraf"), 0o644) - shutil.copyfile(DEFAULT_CONFIG, os.path.join(build_root, CONFIG_DIR[1:], "telegraf.conf")) - os.chmod(os.path.join(build_root, CONFIG_DIR[1:], "telegraf.conf"), 0o644) + if windows: + shutil.copyfile(DEFAULT_WINDOWS_CONFIG, os.path.join(build_root, "telegraf.conf")) + os.chmod(os.path.join(build_root, "telegraf.conf"), 0o644) + else: + shutil.copyfile(INIT_SCRIPT, os.path.join(build_root, SCRIPT_DIR[1:], INIT_SCRIPT.split('/')[1])) + os.chmod(os.path.join(build_root, SCRIPT_DIR[1:], INIT_SCRIPT.split('/')[1]), 0o644) + shutil.copyfile(SYSTEMD_SCRIPT, os.path.join(build_root, SCRIPT_DIR[1:], SYSTEMD_SCRIPT.split('/')[1])) + os.chmod(os.path.join(build_root, SCRIPT_DIR[1:], SYSTEMD_SCRIPT.split('/')[1]), 0o644) + shutil.copyfile(LOGROTATE_SCRIPT, os.path.join(build_root, LOGROTATE_DIR[1:], "telegraf")) + os.chmod(os.path.join(build_root, LOGROTATE_DIR[1:], "telegraf"), 0o644) + shutil.copyfile(DEFAULT_CONFIG, os.path.join(build_root, CONFIG_DIR[1:], "telegraf.conf")) + os.chmod(os.path.join(build_root, CONFIG_DIR[1:], "telegraf.conf"), 0o644) def go_get(): print("Retrieving Go dependencies...") run("go get github.com/sparrc/gdm") + run("gdm restore -f Godeps_windows") run("gdm restore") def generate_md5_from_file(path): @@ -395,15 +401,18 @@ def build_packages(build_output, version, pkg_arch, nightly=False, rc=None, iter build_root = os.path.join(tmp_build_dir, p, a) # Create directory tree to mimic file system of package create_dir(build_root) - create_package_fs(build_root) - # Copy in packaging and miscellaneous scripts - package_scripts(build_root) + if p == 'windows': + package_scripts(build_root, windows=True) + else: + create_package_fs(build_root) + # Copy in packaging and miscellaneous scripts + package_scripts(build_root) # Copy newly-built binaries to packaging directory for b in targets: if p == 'windows': b = b + '.exe' fr = os.path.join(current_location, b) - to = os.path.join(build_root, INSTALL_ROOT_DIR[1:], b) + to = os.path.join(build_root, b) print("\t- [{}][{}] - Moving from '{}' to '{}'".format(p, a, fr, to)) copy_file(fr, to) # Package the directory structure @@ -431,34 +440,44 @@ def build_packages(build_output, version, pkg_arch, nightly=False, rc=None, iter a = pkg_arch if a == '386': a = 'i386' - fpm_command = "fpm {} --name {} -a {} -t {} --version {} --iteration {} -C {} -p {} ".format( - fpm_common_args, - name, - a, - package_type, - package_version, - package_iteration, - build_root, - current_location) - if pkg_arch is not None: - a = saved_a - if package_type == "rpm": - fpm_command += "--depends coreutils " - fpm_command += "--depends lsof" - out = run(fpm_command, shell=True) - matches = re.search(':path=>"(.*)"', out) - outfile = None - if matches is not None: - outfile = matches.groups()[0] - if outfile is None: - print("[ COULD NOT DETERMINE OUTPUT ]") - else: - # Strip nightly version (the unix epoch) from filename - if nightly and package_type in ['deb', 'rpm']: - outfile = rename_file(outfile, outfile.replace("{}-{}".format(version, iteration), "nightly")) - outfiles.append(os.path.join(os.getcwd(), outfile)) - # Display MD5 hash for generated package + if package_type == 'zip': + zip_command = "cd {} && zip {}.zip ./*".format( + build_root, + name) + run(zip_command, shell=True) + run("mv {}.zip {}".format(os.path.join(build_root, name), current_location), shell=True) + outfile = os.path.join(current_location, name+".zip") + outfiles.append(outfile) print("\t\tMD5 = {}".format(generate_md5_from_file(outfile))) + else: + fpm_command = "fpm {} --name {} -a {} -t {} --version {} --iteration {} -C {} -p {} ".format( + fpm_common_args, + name, + a, + package_type, + package_version, + package_iteration, + build_root, + current_location) + if pkg_arch is not None: + a = saved_a + if package_type == "rpm": + fpm_command += "--depends coreutils " + fpm_command += "--depends lsof" + out = run(fpm_command, shell=True) + matches = re.search(':path=>"(.*)"', out) + outfile = None + if matches is not None: + outfile = matches.groups()[0] + if outfile is None: + print("[ COULD NOT DETERMINE OUTPUT ]") + else: + # Strip nightly version (the unix epoch) from filename + if nightly and package_type in ['deb', 'rpm']: + outfile = rename_file(outfile, outfile.replace("{}-{}".format(version, iteration), "nightly")) + outfiles.append(os.path.join(os.getcwd(), outfile)) + # Display MD5 hash for generated package + print("\t\tMD5 = {}".format(generate_md5_from_file(outfile))) print("") if debug: print("[DEBUG] package outfiles: {}".format(outfiles)) diff --git a/scripts/circle-test.sh b/scripts/circle-test.sh index 863fc396f..91511b050 100755 --- a/scripts/circle-test.sh +++ b/scripts/circle-test.sh @@ -76,5 +76,6 @@ if [ $? -eq 0 ]; then tag=$(git describe --exact-match HEAD) echo $tag exit_if_fail ./scripts/build.py --package --version=$tag --platform=linux --arch=all --upload + exit_if_fail ./scripts/build.py --package --version=$tag --platform=windows --arch=all --upload mv build $CIRCLE_ARTIFACTS fi