why does catkin_make keep rebuilding my package from scratch when no changes in src?

edit

I have a "dummy" ROS package that I use to build and source a python virtual environment for my workspace ROS application. there's a shell script that installs pip and installs some packages and another for activating the environment when the workspace is sourced (via env hooks).

This all works, except that every time I run catkin_make, the whole package rebuilds from scratch - so all the pip packages are downloaded and built again - taking a lot of time. Other packages only download and build if there have been changes to the source. Why does this happen and how can I fix it?

virtualenv_build.sh

#!/usr/bin/env bash

# install the venv package if not already present
pip2 install --user virtualenv

# OLDPWD resolves as the workspace build folder when built with catkin
VENV_PATH="$OLDPWD/aios-env"
virtualenv -p "/usr/bin/python2" --system-site-packages "$VENV_PATH"

# source the venv and install packages
. "$VENV_PATH/bin/activate"
python2 -m pip install --upgrade pip
pip install --ignore-installed -r requirements.txt

# if cuda is installed then install the cuda-only packages
if type nvcc; then
  pip install --ignore-installed -r requirements-cuda.txt
fi

virtualenv_source.sh

AIOS_ENV=./build/aios-env
. $AIOS_ENV/bin/activate
# make sure our packages are imported first
export PYTHON_PATH="$AIOS_ENV/lib/python2.7/site-packages/:$PYTHON_PATH"

CMakeLists.sh

cmake_minimum_required(VERSION 3.10.2)
project(virtualenv_config)

set(CMAKE_CXX_STANDARD 14)

find_package(catkin REQUIRED)

# run virtualenv_build.sh at build time
add_custom_target(
  VirtualEnv ALL
  DEPENDS my-env
)
add_custom_command(OUTPUT my-env
                   COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/virtualenv_build.sh
                   WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/)
catkin_package()

# run virtualenv_source.sh when workplace is sourced
catkin_add_env_hooks(50.virtualenv_source SHELLS bash DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/env-hooks)

install(FILES virtualenv_source.sh
  DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION})

install(DIRECTORY my-env/
  DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}/my-env)