Create a separate virtualenv-osx virtualenv for Mac OS X

StackStorm · May 29, 2018 · 7198031 · 7198031
1 parent e961ed5
commit 7198031
Show file tree

Hide file tree

Showing 12 changed files with 124 additions and 53 deletions.
diff --git a/.agignore b/.agignore
@@ -1,2 +1,3 @@
 virtualenv
+virtualenv-osx
 node_modules
diff --git a/.gitignore b/.gitignore
@@ -24,6 +24,7 @@ develop-eggs
 .installed.cfg
 lib64
 virtualenv
+virtualenv-osx
 
 # Installer logs
 pip-log.txt
@@ -42,6 +43,7 @@ nosetests.xml
 .vscode
 *.sublime-project
 *.sublime-workspace
+.envrc
 
 # Editor Saves
 *~

diff --git a/Makefile b/Makefile
@@ -1,7 +1,14 @@
 ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 SHELL := /bin/bash
 TOX_DIR := .tox
-VIRTUALENV_DIR ?= virtualenv
+OS := $(shell uname)
+# We separate the OSX X and Linux virtualenvs so we can run in a Docker
+# container (st2devbox) while doing things on our host Mac machine
+ifeq ($(OS),Darwin)
+	VIRTUALENV_DIR ?= virtualenv-osx
+else
+	VIRTUALENV_DIR ?= virtualenv
+endif
 PYTHON_VERSION = python2.7
 
 BINARIES := bin
@@ -294,34 +301,57 @@ virtualenv:
 	@echo
 	# Note: We pass --no-download flag to make sure version of pip which we install (9.0.1) is used
 	# instead of latest version being downloaded from PyPi
-	test -f $(VIRTUALENV_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) --no-site-packages $(VIRTUALENV_DIR) --no-download
-
-	# Setup PYTHONPATH in bash activate script...
-	# Delete existing entries (if any)
-	sed -i '/_OLD_PYTHONPATHp/d' $(VIRTUALENV_DIR)/bin/activate
-	sed -i '/PYTHONPATH=/d' $(VIRTUALENV_DIR)/bin/activate
-	sed -i '/export PYTHONPATH/d' $(VIRTUALENV_DIR)/bin/activate
-
-	echo '_OLD_PYTHONPATH=$$PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate
-	#echo 'PYTHONPATH=$$_OLD_PYTHONPATH:$(COMPONENT_PYTHONPATH)' >> $(VIRTUALENV_DIR)/bin/activate
-	echo 'PYTHONPATH=${ROOT_DIR}:$(COMPONENT_PYTHONPATH)' >> $(VIRTUALENV_DIR)/bin/activate
-	echo 'export PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate
-	touch $(VIRTUALENV_DIR)/bin/activate
-
-	# Setup PYTHONPATH in fish activate script...
-	#echo '' >> $(VIRTUALENV_DIR)/bin/activate.fish
-	#echo 'set -gx _OLD_PYTHONPATH $$PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate.fish
-	#echo 'set -gx PYTHONPATH $$_OLD_PYTHONPATH $(COMPONENT_PYTHONPATH)' >> $(VIRTUALENV_DIR)/bin/activate.fish
-	#echo 'functions -c deactivate old_deactivate' >> $(VIRTUALENV_DIR)/bin/activate.fish
-	#echo 'function deactivate' >> $(VIRTUALENV_DIR)/bin/activate.fish
-	#echo '  if test -n $$_OLD_PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate.fish
-	#echo '    set -gx PYTHONPATH $$_OLD_PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate.fish
-	#echo '    set -e _OLD_PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate.fish
-	#echo '  end' >> $(VIRTUALENV_DIR)/bin/activate.fish
-	#echo '  old_deactivate' >> $(VIRTUALENV_DIR)/bin/activate.fish
-	#echo '  functions -e old_deactivate' >> $(VIRTUALENV_DIR)/bin/activate.fish
-	#echo 'end' >> $(VIRTUALENV_DIR)/bin/activate.fish
-	#touch $(VIRTUALENV_DIR)/bin/activate.fish
+	# For OS X, we have to bootstrap install pip because otherwise it runs into
+	# SSLv1 errors when connecting to https://pypi.python.org/simple/ to
+	# download anything
+	@if [[ "$(OS)" == "Darwin" ]]; then \
+	    if [[ ! -f $(VIRTUALENV_DIR)/bin/activate ]]; then \
+	        virtualenv --python=$(PYTHON_VERSION) --no-site-packages --no-setuptools --never-download $(VIRTUALENV_DIR); \
+	        echo "Setup PYTHONPATH in bash activate script..."; \
+	        echo "Delete existing entries (if any)"; \
+	        sed -i'' -e '/_OLD_PYTHONPATHp/d' $(VIRTUALENV_DIR)/bin/activate; \
+	        sed -i'' -e '/PYTHONPATH=/d' $(VIRTUALENV_DIR)/bin/activate; \
+	        sed -i'' -e '/export PYTHONPATH/d' $(VIRTUALENV_DIR)/bin/activate; \
+	        echo '_OLD_PYTHONPATH=$$PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate; \
+	        echo "#echo 'PYTHONPATH=$$_OLD_PYTHONPATH:$(COMPONENT_PYTHONPATH)' >> $(VIRTUALENV_DIR)/bin/activate"; \
+	        echo 'PYTHONPATH=${ROOT_DIR}:$(COMPONENT_PYTHONPATH)' >> $(VIRTUALENV_DIR)/bin/activate; \
+	        echo 'export PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate; \
+	        touch $(VIRTUALENV_DIR)/bin/activate; \
+	        source $(VIRTUALENV_DIR)/bin/activate; \
+	        echo "Installing pip"; \
+	        curl --silent https://bootstrap.pypa.io/get-pip.py | python; \
+	        python -c 'import setuptools; print setuptools.__version__'; \
+	    fi; \
+	else \
+	    if [[ ! -f $(VIRTUALENV_DIR)/bin/activate ]]; then \
+	        virtualenv --python=$(PYTHON_VERSION) --no-site-packages --no-download $(VIRTUALENV_DIR); \
+	        echo "Setup PYTHONPATH in bash activate script..."; \
+	        echo "Delete existing entries (if any)"; \
+	        sed -i '/_OLD_PYTHONPATHp/d' $(VIRTUALENV_DIR)/bin/activate; \
+	        sed -i '/PYTHONPATH=/d' $(VIRTUALENV_DIR)/bin/activate; \
+	        sed -i '/export PYTHONPATH/d' $(VIRTUALENV_DIR)/bin/activate; \
+	        echo '_OLD_PYTHONPATH=$$PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate; \
+	        echo "#echo 'PYTHONPATH=$$_OLD_PYTHONPATH:$(COMPONENT_PYTHONPATH)' >> $(VIRTUALENV_DIR)/bin/activate"; \
+	        echo 'PYTHONPATH=${ROOT_DIR}:$(COMPONENT_PYTHONPATH)' >> $(VIRTUALENV_DIR)/bin/activate; \
+	        echo 'export PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate; \
+	        touch $(VIRTUALENV_DIR)/bin/activate; \
+	    fi; \
+	fi
+
+	@# Setup PYTHONPATH in fish activate script...
+	@#echo '' >> $(VIRTUALENV_DIR)/bin/activate.fish
+	@#echo 'set -gx _OLD_PYTHONPATH $$PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate.fish
+	@#echo 'set -gx PYTHONPATH $$_OLD_PYTHONPATH $(COMPONENT_PYTHONPATH)' >> $(VIRTUALENV_DIR)/bin/activate.fish
+	@#echo 'functions -c deactivate old_deactivate' >> $(VIRTUALENV_DIR)/bin/activate.fish
+	@#echo 'function deactivate' >> $(VIRTUALENV_DIR)/bin/activate.fish
+	@#echo '  if test -n $$_OLD_PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate.fish
+	@#echo '    set -gx PYTHONPATH $$_OLD_PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate.fish
+	@#echo '    set -e _OLD_PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate.fish
+	@#echo '  end' >> $(VIRTUALENV_DIR)/bin/activate.fish
+	@#echo '  old_deactivate' >> $(VIRTUALENV_DIR)/bin/activate.fish
+	@#echo '  functions -e old_deactivate' >> $(VIRTUALENV_DIR)/bin/activate.fish
+	@#echo 'end' >> $(VIRTUALENV_DIR)/bin/activate.fish
+	@#touch $(VIRTUALENV_DIR)/bin/activate.fish
 
 .PHONY: tests
 tests: pytests
@@ -400,6 +430,7 @@ itests: requirements .itests
 			--cover-inclusive --cover-html \
 			--cover-package=$(COMPONENTS_TEST_COMMA) $$component/tests/integration || exit 1; \
 	done
+	@. $(VIRTUALENV_DIR)/bin/activate; coverage html
 
 .PHONY: mistral-itests
 mistral-itests: requirements .mistral-itests

diff --git a/st2actions/Makefile b/st2actions/Makefile
@@ -1,7 +1,12 @@
 # <<<< TO DEPRICATE
 SHELL := /bin/bash
 REPO_ROOT := ..
-VIRTUALENV_DIR=virtualenv
+OS := $(shell uname)
+ifeq ($(OS),Darwin)
+	VIRTUALENV_DIR ?= virtualenv-osx
+else
+	VIRTUALENV_DIR ?= virtualenv
+endif
 RPM_ROOT=~/rpmbuild
 RPM_SOURCES_DIR := $(RPM_ROOT)/SOURCES/
 RPM_SPECS_DIR := $(RPM_ROOT)/SPECS/

diff --git a/st2api/Makefile b/st2api/Makefile
@@ -1,7 +1,12 @@
 # <<<< TO DEPRICATE
 SHELL := /bin/bash
 REPO_ROOT := ..
-VIRTUALENV_DIR=virtualenv
+OS := $(shell uname)
+ifeq ($(OS),Darwin)
+	VIRTUALENV_DIR ?= virtualenv-osx
+else
+	VIRTUALENV_DIR ?= virtualenv
+endif
 RPM_ROOT=~/rpmbuild
 RPM_SOURCES_DIR := $(RPM_ROOT)/SOURCES/
 RPM_SPECS_DIR := $(RPM_ROOT)/SPECS/

diff --git a/st2auth/Makefile b/st2auth/Makefile
@@ -1,7 +1,12 @@
 # <<<< TO DEPRICATE
 SHELL := /bin/bash
 REPO_ROOT := ..
-VIRTUALENV_DIR=virtualenv
+OS := $(shell uname)
+ifeq ($(OS),Darwin)
+	VIRTUALENV_DIR ?= virtualenv-osx
+else
+	VIRTUALENV_DIR ?= virtualenv
+endif
 RPM_ROOT=~/rpmbuild
 RPM_SOURCES_DIR := $(RPM_ROOT)/SOURCES/
 RPM_SPECS_DIR := $(RPM_ROOT)/SPECS/

diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py
@@ -15,6 +15,7 @@
 
 from __future__ import absolute_import
 import os
+import platform
 import socket
 import sys
 
@@ -230,7 +231,8 @@ def register_opts(ignore_errors=False):
     default_python_bin_path = sys.executable
     default_python3_bin_path = find_executable('python3')
     base_dir = os.path.dirname(os.path.realpath(default_python_bin_path))
-    default_virtualenv_bin_path = os.path.join(base_dir, 'virtualenv')
+    virtualenv_dir_name = 'virtualenv-osx' if platform.system() == "Darwin" else 'virtualenv'
+    default_virtualenv_bin_path = os.path.join(base_dir, virtualenv_dir_name)
 
     action_runner_opts = [
         # Common runner options

diff --git a/st2debug/Makefile b/st2debug/Makefile
@@ -1,7 +1,12 @@
 # <<<< TO DEPRICATE
 SHELL := /bin/bash
 REPO_ROOT := ..
-VIRTUALENV_DIR=virtualenv
+OS := $(shell uname)
+ifeq ($(OS),Darwin)
+	VIRTUALENV_DIR ?= virtualenv-osx
+else
+	VIRTUALENV_DIR ?= virtualenv
+endif
 RPM_ROOT=~/rpmbuild
 RPM_SOURCES_DIR := $(RPM_ROOT)/SOURCES/
 RPM_SPECS_DIR := $(RPM_ROOT)/SPECS/

diff --git a/st2tests/Makefile b/st2tests/Makefile
@@ -1,7 +1,12 @@
 # <<<< TO DEPRICATE
 SHELL := /bin/bash
 REPO_ROOT := ..
-VIRTUALENV_DIR=virtualenv
+OS := $(shell uname)
+ifeq ($(OS),Darwin)
+	VIRTUALENV_DIR ?= virtualenv-osx
+else
+	VIRTUALENV_DIR ?= virtualenv
+endif
 RPM_ROOT=~/rpmbuild
 RPM_SOURCES_DIR := $(RPM_ROOT)/SOURCES/
 RPM_SPECS_DIR := $(RPM_ROOT)/SPECS/

diff --git a/st2tests/integration/run-ssh-tests.sh b/st2tests/integration/run-ssh-tests.sh
@@ -64,7 +64,12 @@ function sshtest(){
     echo "Running SSH tests in a loop(count=$loop_count) ..."
     echo "Activating virtual environment..."
     # activate virtualenv to set PYTHONPATH
-    source ./virtualenv/bin/activate
+    if [[ "$(uname 2>/dev/null)" == "Darwin" ]]; then
+        VIRTUALENV_DIR=virtualenv-osx
+    else
+        VIRTUALENV_DIR=virtualenv
+    fi
+    source ./${VIRTUALENV_DIR}/bin/activate
 
     # Run SSH commands test.
     for i in `seq 1 ${loop_count}`; do

diff --git a/tools/launchdev.sh b/tools/launchdev.sh
@@ -1,5 +1,12 @@
 #!/usr/bin/env bash
 
+if [[ "$(uname 2>/dev/null)" == "Darwin" ]]; then
+    VIRTUALENV_DIR=virtualenv-osx
+else
+    VIRTUALENV_DIR=virtualenv
+fi
+VENV=${ST2_REPO}/${VIRTUALENV_DIR}
+
 function usage() {
     echo "Usage: $0 [start|stop|restart|startclean] [-r runner_count] [-g] [-x] [-c] [-6] [-m]" >&2
 }
@@ -58,8 +65,6 @@ function init(){
     else
         ST2_REPO=${CURRENT_DIR}/${COMMAND_PATH}/..
     fi
-
-    VENV=${ST2_REPO}/virtualenv
     PY=${VENV}/bin/python
     echo "Using virtualenv: ${VENV}"
     echo "Using python: ${PY}"
@@ -215,7 +220,7 @@ function st2start(){
     fi
 
     # activate virtualenv to set PYTHONPATH
-    source ./virtualenv/bin/activate
+    source ${VENV}/bin/activate
 
     # Kill existing st2 screens
     screen -wipe
@@ -236,10 +241,10 @@ function st2start(){
     if [ "${use_gunicorn}" = true ]; then
         echo '  using gunicorn to run st2-api...'
         export ST2_CONFIG_PATH=${ST2_CONF}
-        screen -d -m -S st2-api ./virtualenv/bin/gunicorn \
+        screen -d -m -S st2-api ${VENV}/bin/gunicorn \
             st2api.wsgi:application -k eventlet -b "$BINDING_ADDRESS:9101" --workers 1
     else
-        screen -d -m -S st2-api ./virtualenv/bin/python \
+        screen -d -m -S st2-api ${VENV}/bin/python \
             ./st2api/bin/st2api \
             --config-file $ST2_CONF
     fi
@@ -248,10 +253,10 @@ function st2start(){
     if [ "${use_gunicorn}" = true ]; then
         echo '  using gunicorn to run st2-stream'
         export ST2_CONFIG_PATH=${ST2_CONF}
-        screen -d -m -S st2-stream ./virtualenv/bin/gunicorn \
+        screen -d -m -S st2-stream ${VENV}/bin/gunicorn \
             st2stream.wsgi:application -k eventlet -b "$BINDING_ADDRESS:9102" --workers 1
     else
-        screen -d -m -S st2-stream ./virtualenv/bin/python \
+        screen -d -m -S st2-stream ${VENV}/bin/python \
             ./st2stream/bin/st2stream \
             --config-file $ST2_CONF
     fi
@@ -264,32 +269,32 @@ function st2start(){
         RUNNER_NAME=st2-actionrunner-$i
         RUNNER_SCREENS+=($RUNNER_NAME)
         echo '  starting '$RUNNER_NAME'...'
-        screen -d -m -S $RUNNER_NAME ./virtualenv/bin/python \
+        screen -d -m -S $RUNNER_NAME ${VENV}/bin/python \
             ./st2actions/bin/st2actionrunner \
             --config-file $ST2_CONF
     done
 
     # Run the sensor container server
     echo 'Starting screen session st2-sensorcontainer'
-    screen -d -m -S st2-sensorcontainer ./virtualenv/bin/python \
+    screen -d -m -S st2-sensorcontainer ${VENV}/bin/python \
         ./st2reactor/bin/st2sensorcontainer \
         --config-file $ST2_CONF
 
     # Run the rules engine server
     echo 'Starting screen session st2-rulesengine...'
-    screen -d -m -S st2-rulesengine ./virtualenv/bin/python \
+    screen -d -m -S st2-rulesengine ${VENV}/bin/python \
         ./st2reactor/bin/st2rulesengine \
         --config-file $ST2_CONF
 
     # Run the results tracker
     echo 'Starting screen session st2-resultstracker...'
-    screen -d -m -S st2-resultstracker ./virtualenv/bin/python \
+    screen -d -m -S st2-resultstracker ${VENV}/bin/python \
         ./st2actions/bin/st2resultstracker \
         --config-file $ST2_CONF
 
     # Run the actions notifier
     echo 'Starting screen session st2-notifier...'
-    screen -d -m -S st2-notifier ./virtualenv/bin/python \
+    screen -d -m -S st2-notifier ${VENV}/bin/python \
         ./st2actions/bin/st2notifier \
         --config-file $ST2_CONF
 
@@ -298,10 +303,10 @@ function st2start(){
     if [ "${use_gunicorn}" = true ]; then
         echo '  using gunicorn to run st2-auth...'
         export ST2_CONFIG_PATH=${ST2_CONF}
-        screen -d -m -S st2-auth ./virtualenv/bin/gunicorn \
+        screen -d -m -S st2-auth ${VENV}/bin/gunicorn \
             st2auth.wsgi:application -k eventlet -b "$BINDING_ADDRESS:9100" --workers 1
     else
-        screen -d -m -S st2-auth ./virtualenv/bin/python \
+        screen -d -m -S st2-auth ${VENV}/bin/python \
             ./st2auth/bin/st2auth \
             --config-file $ST2_CONF
     fi
@@ -312,7 +317,7 @@ function st2start(){
         sudo mkdir -p $EXPORTS_DIR
         sudo chown -R ${CURRENT_USER}:${CURRENT_USER_GROUP} $EXPORTS_DIR
         echo 'Starting screen session st2-exporter...'
-        screen -d -m -S st2-exporter ./virtualenv/bin/python \
+        screen -d -m -S st2-exporter ${VENV}/bin/python \
             ./st2exporter/bin/st2exporter \
             --config-file $ST2_CONF
     fi
@@ -366,7 +371,7 @@ function st2start(){
     if [ "$load_content" = true ]; then
         # Register contents
         echo 'Registering sensors, runners, actions, rules, aliases, and policies...'
-        ./virtualenv/bin/python \
+        ${VENV}/bin/python \
             ./st2common/bin/st2-register-content \
             --config-file $ST2_CONF --register-all
     fi

diff --git a/tools/log_watcher.py b/tools/log_watcher.py
@@ -70,7 +70,7 @@ def _get_files(dir_path):
         print('Directory %s doesn\'t exist.' % dir_path)
 
     files = []
-    exclude = set(['virtualenv', 'build', '.tox'])
+    exclude = set(['virtualenv', 'virtualenv-osx', 'build', '.tox'])
     for root, dirnames, filenames in os.walk(dir_path):
         dirnames[:] = [d for d in dirnames if d not in exclude]
         for filename in fnmatch.filter(filenames, '*.py'):