Revision: 15479 http://trac.calendarserver.org//changeset/15479 Author: sagen@apple.com Date: 2016-03-18 17:26:34 -0700 (Fri, 18 Mar 2016) Log Message: ----------- Git based submission and dependency-caching scripts Added Paths: ----------- CalendarServer/trunk/support/_cache_deps_git CalendarServer/trunk/support/submit_git Added: CalendarServer/trunk/support/_cache_deps_git =================================================================== --- CalendarServer/trunk/support/_cache_deps_git (rev 0) +++ CalendarServer/trunk/support/_cache_deps_git 2016-03-19 00:26:34 UTC (rev 15479) @@ -0,0 +1,149 @@ +#!/bin/sh +# -*- sh-basic-offset: 2 -*- + +## +# Copyright (c) 2005-2016 Apple Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## + +set -e +set -u + +wd="$(cd "$(dirname "$0")/.." && pwd)"; + +. "${wd}/bin/_build.sh"; + +init_build; + +export PATH="/Applications/Server.app/Contents/ServerRoot/usr/bin:${PATH}"; + +requirements="${wd}/requirements-osx.txt"; +extra_features="OpenDirectory,Postgres"; + + +# +# Download virtualenv and friends so that B&I has that core toolchain. +# + +mkdir -p "${wd}/.develop/tools"; + +for pkg in \ + setuptools-18.5 \ + pip-7.1.2 \ + virtualenv-13.1.2 \ +; do + name="${pkg%-*}"; + version="${pkg#*-}"; + first="$(echo "${name}" | sed 's|^\(.\).*$|\1|')"; + url="https://pypi.python.org/packages/source/${first}/${name}/${pkg}.tar.gz"; + + ruler "Downloading ${pkg}"; + + curl -L -o "${wd}/.develop/tools/${pkg}.tgz" "${url}"; +done; + + +# +# Build cffi because xattr needs it at setup time. +# + +if ! find_header ffi.h; then + c_glue_include="${dev_roots}/c_glue/include"; + mkdir -p "${c_glue_include}"; + echo "#include <ffi/ffi.h>" > "${c_glue_include}/ffi.h" + export C_INCLUDE_PATH="${c_glue_include}:${C_INCLUDE_PATH:-}"; +fi; + + +# +# Download dependencies +# + +ve_tmp="$(mktemp -d -t CalendarServer_ve_tools)"; + +# Bootstrap virtualenv and friends so we can use them in this script (not sent to B&I). +py_ve_tools="${ve_tmp}/ve_tools"; +export PYTHONPATH="${py_ve_tools}/lib:${wd}:${PYTHONPATH:-}"; +bootstrap_virtualenv; + +reqs_to_use="${ve_tmp}/requirements-osx.txt"; + +# Copy over each requirements file we need and edit a few to remove stuff we don't need +sed -e '/requirements-dev.txt/d' < "${wd}/requirements-osx.txt" > "${ve_tmp}/requirements-osx.txt"; +sed -e '/python-ldap/d' < "${wd}/requirements-cs.txt" > "${ve_tmp}/requirements-cs.txt"; +echo "xattr==0.7.5" >> "${ve_tmp}/requirements-cs.txt"; +cp "${wd}/requirements-twisted-osx.txt" "${ve_tmp}/"; + +ruler "Downloading Python requirements for .[${extra_features}]"; +echo ""; +pip_download \ + --allow-unverified cx-Oracle \ + --no-deps \ + --requirement="${reqs_to_use}" \ + ; + +rm -rf "${ve_tmp}"; + + +# +# Check out CalDAVTester +# + +url="$(grep egg=CalDAVTester "${wd}/requirements-dev.txt" | sed 's|^.*git+\([^@#]*\).*$|\1|')"; +rev="$(grep egg=CalDAVTester "${wd}/requirements-dev.txt" | sed 's|^.*git+[^@#]*@\([0-9a-z]*\).*$|\1|')"; + +git clone "${url}" "${wd}/CalDAVTester"; +rm -rf "${wd}/CalDAVTester/.git"; +# FIXME: checkout rev +tar -C "${wd}" -cvzf "${wd}/CalDAVTester.tgz" CalDAVTester; +rm -r CalDAVTester; + + + +# +# Remove .exe files from archives +# + +for archive in $(find "${wd}/.develop" -type f -name '*.tgz' -or -name '*.tar.gz'); do + if tar -tvzf "${archive}" "*.exe" > /dev/null 2>&1; then + ruler "Removing binaries from ${archive}"; + tmp="$(mktemp -t ccsXXXXX)"; + gzcat "${archive}" | gnutar --delete --wildcards -vf - "*.exe" > "${tmp}"; + gzip -c "${tmp}" > "${archive}"; + rm "${tmp}"; + fi; +done; + + + +# +# Remove .eggs files from archives +# + +for archive in $(find "${wd}/.develop" -type f -name '*.zip'); do + if unzip -qql "${archive}" "*/.eggs/" > /dev/null 2>&1; then + ruler "Removing eggs from ${archive}"; + zip -qd "${archive}" "*/.eggs/*"; + fi; +done; + + + +# +# Remove src +# +ruler "Removing eggs from ${archive}"; +rm -rf "${wd}/src" + +ruler ""; Property changes on: CalendarServer/trunk/support/_cache_deps_git ___________________________________________________________________ Added: svn:executable + * Added: CalendarServer/trunk/support/submit_git =================================================================== --- CalendarServer/trunk/support/submit_git (rev 0) +++ CalendarServer/trunk/support/submit_git 2016-03-19 00:26:34 UTC (rev 15479) @@ -0,0 +1,221 @@ +#!/bin/sh + +## +# Copyright (c) 2005-2016 Apple Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## + +## +# Submit project to B&I (Apple internal build) +## + +set -e +set -u + + + wd="$(cd "$(dirname "$0")" && pwd)"; +src="$(cd "${wd}/.." && pwd)"; + +unset CALENDARSERVER_BUILD_DEPS; + +project="CalendarServer"; +uri="https://github.com/macosforge/ccs-calendarserver-copy.git"; + +## +# Command line +## + + build=false; +install=false; +package=false; + + build_no_verify_source=false; + tag_to_build=""; + project_version=""; + +usage () +{ + program="$(basename "$0")"; + + if [ "${1-}" != "-" ]; then echo "$@"; echo; fi; + + echo "Usage: ${program} release [release ...]"; + echo " ${program} release -b[ipn]"; + echo ""; + echo "Options:"; + echo " -b Run buildit"; + echo " -i Install resulting build on this system"; + echo " -p Create a package with the resulting build"; + echo " -n skip buildit source verification"; + echo " -t which tag to submit"; + echo " -v project version (defaults to -t value)"; + + if [ "${1-}" == "-" ]; then return 0; fi; + exit 64; +} + +while getopts 'hbipnt:v:' option; do + case "$option" in + '?') usage; ;; + 'h') usage -; exit 0; ;; + 'b') build=true; ;; + 'i') install=true; ;; + 'p') package=true; ;; + 'n') build_no_verify_source=true; ;; + 't') tag_to_build="${OPTARG}"; ;; + 'v') project_version="${OPTARG}"; ;; + esac; +done; +shift $((${OPTIND} - 1)); + +if [ "${tag_to_build}" != "" ]; then + if [ "${project_version}" == "" ]; then + project_version="${tag_to_build}"; + fi; +fi; + + +if ! "${build}"; then + # Submitting, not just building + if "${install}"; then usage "-i flag requires -b"; fi; + if "${package}"; then usage "-p flag requires -b"; fi; + if "${build_no_verify_source}"; then usage "-n flag requires -b"; fi; + if [ "${project_version}" == "" ]; then usage "project_version required if submitting"; fi; +else + # Not submitting, just building + if [ "${project_version}" == "" ]; then + project_version="${project}-untagged"; + fi; +fi; + +if [ $# == 0 ]; then usage "No releases specified"; fi; +releases="$@"; shift $#; + +if [ $# != 0 ]; then usage "Unrecognized arguments:" "$@"; fi; + +## +# Do the Right Thing +## + +# +# Do submission +# + +tmp="$(mktemp -d -t CalendarServer_build)"; +wc="${tmp}/${project_version}"; + +echo "${tag_to_build}"; + +if [ "${tag_to_build}" != "" ]; then + # Clone and checkout tag + git clone "${uri}" "${wc}"; + cd "${wc}"; + git checkout "${tag_to_build}"; +else + # Copy from local directory + echo ""; + echo "Copying ${src}..."; + # Only copy files that are under revision control + git_files="$(mktemp -t git_files_XXXX)"; + cd "${src}"; + git ls-tree --name-only -r HEAD > "${git_files}"; + rsync -av \ + --files-from="${git_files}" \ + "${src}/" "${wc}" \ + ; + rm "${git_files}"; +fi; + +cd "${wc}"; + +echo "" +echo "Tweaking for B&I..."; +ln -s support/Apple.make "${wc}/Makefile"; + +cat - >> "${wc}/SubmissionInfo.xml" <<EOF +<?xml version="1.0" encoding="UTF-8"?> +<submission> + <project>${project}</project> + <version>${project_version}</version> + <source> + <git> + <repository>${uri}</repository> + <tag>${tag_to_build}</tag> + <date>$(date -u)</date> + </git> + </source> +</submission> +EOF + +echo "${tag_to_build}" > "${wc}/gitversion.txt"; + +echo ""; +echo "Preparing sources for ${project_version}..."; +"${wc}/support/_cache_deps_git"; + +# Clean up after _cache_deps +find "${wc}/.develop" -depth 1 ! '(' -name pip_downloads -o -name tools ')' -print | { + while read filename; do + echo "Cruft in .develop: ${filename}"; + # exit 1; + done; +} + +if "${build}"; then + echo ""; + echo "Building ${project_version}..."; + + if "${package}"; then + package_tmp="${tmp}/pkg"; + install -d "${package_tmp}"; + merge_flags="-merge ${package_tmp}"; + elif "${install}"; then + merge_flags="-merge /"; + else + merge_flags=""; + fi; + + release_flags=""; + for release in "${releases}"; do + release_flags="${release_flags} -update Prevailing${release}"; + done; + + if "${build_no_verify_source}"; then + verify_flags=" -noverifysource"; + else + verify_flags=""; + fi; + + sudo ~rc/bin/buildit "${wc}" \ + $(file /System/Library/Frameworks/Python.framework/Versions/Current/Python | sed -n -e 's|^.*(for architecture \([^)][^)]*\).*$|-arch \1|p' | sed 's|ppc7400|ppc|') \ + ${merge_flags}${release_flags}${verify_flags}; + + if "${package}"; then + package_file="${src}/${project_version}.tgz"; + echo "Creating package: ${package_file}..."; + tar -C "${package_tmp}" -cvzf "${package_file}" .; + sudo rm -rf "${package_tmp}"; + if "${install}"; then + echo "Installing package: ${package_file}"; + tar -C / -xvzf "${package_file}"; + fi; + fi; +else + echo ""; + echo "Submitting sources for ${project_version}..."; + rm -rf "${wc}/.dependencies"; + ~rc/bin/submitproject "${wc}" ${releases}; +fi; + +sudo rm -rf "${tmp}"; Property changes on: CalendarServer/trunk/support/submit_git ___________________________________________________________________ Added: svn:executable + *