feature: install sagemaker-tensorflow-toolkit from PyPI. (#335)

Author: nadiaya

README.rst

@@ -114,17 +114,6 @@ Before building "final" images:
 Build your "base" image. Make sure it is named and tagged in accordance with your "final"
 Dockerfile. Skip this step if you want to build image of Tensorflow Version 1.9.0 and above.
 
-Then prepare the SageMaker TensorFlow Container python package in the image folder like below:
-
-::
-
-    # Create the SageMaker TensorFlow Container Python package.
-    cd sagemaker-tensorflow-containers
-    python setup.py sdist
-
-    #. Copy your Python package to "final" Dockerfile directory that you are building.
-    cp dist/sagemaker_tensorflow_container-<package_version>.tar.gz docker/<tensorflow_version>/final/py2
-
 If you want to build "final" Docker images, for versions 1.6 and above, you will first need to download the appropriate tensorflow pip wheel, then pass in its location as a build argument. These can be obtained from pypi. For example, the files for 1.6.0 are here:
 
 https://pypi.org/project/tensorflow/1.6.0/#files

buildspec.yml

@@ -23,20 +23,13 @@ phases:
   build:
     commands:
       # install
-      - pip3 install -U -e .
       - pip3 install -U -e .[test]
 
       # run flake8
       - tox -e flake8,twine
 
-      # run unit tests
-      - tox -e py36,py27 test/unit
-
-      # Create pip archive
       - root_dir=$(pwd)
       - build_id="$(echo $CODEBUILD_BUILD_ID | sed -e 's/:/-/g')"
-      - python3 setup.py sdist
-      - tar_name=$(ls dist)
 
       # Find build artifacts
       - build_artifacts=$root_dir/docker/build_artifacts
@@ -48,48 +41,54 @@ phases:
 
       # prepare build context
       - build_dir="$root_dir/docker/$FRAMEWORK_VERSION/py2"
-      - cp $root_dir/dist/$tar_name $build_dir
       - cp $build_artifacts/*.py $build_dir/
       - cd $build_dir
 
       # build cpu image
       - prod_tag="$FRAMEWORK_VERSION-cpu-py2"
       - CPU_TAG_PY2="$prod_tag-$build_id"
       - docker pull $PROD_IMAGE:$prod_tag
-      - docker build --cache-from $PROD_IMAGE:$prod_tag -f $CPU_DOCKERFILE -t $PREPROD_IMAGE:$CPU_TAG_PY2 .
+      - build_cmd="docker build --cache-from $PROD_IMAGE:$prod_tag -f $CPU_DOCKERFILE -t $PREPROD_IMAGE:$CPU_TAG_PY2 . "
+      - execute-command-if-has-matching-changes "$build_cmd" "test/" "docker/*" "buildspec.yml"
 
       # build gpu image
       - prod_tag="$FRAMEWORK_VERSION-gpu-py2"
       - GPU_TAG_PY2="$prod_tag-$build_id"
       - docker pull $PROD_IMAGE:$prod_tag
-      - docker build --cache-from $PROD_IMAGE:$prod_tag -f $GPU_DOCKERFILE -t $PREPROD_IMAGE:$GPU_TAG_PY2 .
+      - build_cmd="docker build --cache-from $PROD_IMAGE:$prod_tag -f $GPU_DOCKERFILE -t $PREPROD_IMAGE:$GPU_TAG_PY2 . "
+      - execute-command-if-has-matching-changes "$build_cmd" "test/" "docker/*" "buildspec.yml"
 
       # build py3 images
 
       # prepare build context
       - build_dir="$root_dir/docker/$FRAMEWORK_VERSION/py3"
-      - cp $root_dir/dist/$tar_name $build_dir
       - cp $build_artifacts/*.py $build_dir/
       - cd $build_dir
 
       # build cpu image
       - prod_tag="$FRAMEWORK_VERSION-cpu-py3"
       - CPU_TAG_PY3="$prod_tag-$build_id"
       - docker pull $PROD_IMAGE:$prod_tag
-      - docker build --cache-from $PROD_IMAGE:$prod_tag -f $CPU_DOCKERFILE -t $PREPROD_IMAGE:$CPU_TAG_PY3 .
+      - build_cmd="docker build --cache-from $PROD_IMAGE:$prod_tag -f $CPU_DOCKERFILE -t $PREPROD_IMAGE:$CPU_TAG_PY3 . "
+      - execute-command-if-has-matching-changes "$build_cmd" "test/" "docker/*" "buildspec.yml"
 
       # build gpu image
       - prod_tag="$FRAMEWORK_VERSION-gpu-py3"
       - GPU_TAG_PY3="$prod_tag-$build_id"
       - docker pull $PROD_IMAGE:$prod_tag
-      - docker build --cache-from $PROD_IMAGE:$prod_tag -f $GPU_DOCKERFILE -t $PREPROD_IMAGE:$GPU_TAG_PY3 .
+      - build_cmd="docker build --cache-from $PROD_IMAGE:$prod_tag -f $GPU_DOCKERFILE -t $PREPROD_IMAGE:$GPU_TAG_PY3 . "
+      - execute-command-if-has-matching-changes "$build_cmd" "test/" "docker/*" "buildspec.yml"
 
       # push images to ecr
       - $(aws ecr get-login --registry-ids $ACCOUNT --no-include-email --region $AWS_DEFAULT_REGION)
       - docker push $PREPROD_IMAGE:$CPU_TAG_PY2
+      - execute-command-if-has-matching-changes "$push_cmd" "test/" "docker/*" "buildspec.yml"
       - docker push $PREPROD_IMAGE:$GPU_TAG_PY2
+      - execute-command-if-has-matching-changes "$push_cmd" "test/" "docker/*" "buildspec.yml"
       - docker push $PREPROD_IMAGE:$CPU_TAG_PY3
+      - execute-command-if-has-matching-changes "$push_cmd" "test/" "docker/*" "buildspec.yml"
       - docker push $PREPROD_IMAGE:$GPU_TAG_PY3
+      - execute-command-if-has-matching-changes "$push_cmd" "test/" "docker/*" "buildspec.yml"
 
       # launch remote gpu instance
       - instance_type='p2.xlarge'
@@ -98,37 +97,41 @@ phases:
 
       # run cpu integration tests
       - py3_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $CPU_TAG_PY3 --framework-version $FRAMEWORK_VERSION --py-version 3 --processor cpu --durations 10"
-      - execute-command-if-has-matching-changes "$py3_cmd" "test/" "src/*.py" "setup.py" "docker/*" "buildspec.yml"
+      - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
       - py2_cmd="IGNORE_COVERAGE=- tox -e py27 -- test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $CPU_TAG_PY2 --framework-version $FRAMEWORK_VERSION --py-version 2 --processor cpu --durations 10"
-      - execute-command-if-has-matching-changes "$py2_cmd" "test/" "src/*.py" "setup.py" "docker/*" "buildspec.yml"
+      - execute-command-if-has-matching-changes "$py2_cmd" "test/" "docker/*" "buildspec.yml"
 
       # run gpu integration tests
       - printf "$SETUP_CMDS" > $SETUP_FILE
       - cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local -n 4 --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $GPU_TAG_PY3 --framework-version $FRAMEWORK_VERSION --py-version 3 --processor gpu --durations 10"
       - py3_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\""
-      - execute-command-if-has-matching-changes "$py3_cmd" "test/" "src/*.py" "setup.py" "docker/*" "buildspec.yml"
+      - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
 
       - cmd="IGNORE_COVERAGE=- tox -e py27 -- test/integration/local -n 4 --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $GPU_TAG_PY2 --framework-version $FRAMEWORK_VERSION --py-version 2 --processor gpu --durations 10"
       - py2_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\" --skip-setup"
-      - execute-command-if-has-matching-changes "$py2_cmd" "test/" "src/*.py" "setup.py" "docker/*" "buildspec.yml"
+      - execute-command-if-has-matching-changes "$py2_cmd" "test/" "docker/*" "buildspec.yml"
 
       # run sagemaker tests
       - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $CPU_TAG_PY3 --py-version 3 --processor cpu --durations 10"
-      - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "docker/*" "buildspec.yml"
+      - execute-command-if-has-matching-changes "$test_cmd" "test/" "docker/*" "buildspec.yml"
       - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $GPU_TAG_PY3 --py-version 3 --processor gpu --durations 10"
-      - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "docker/*" "buildspec.yml"
+      - execute-command-if-has-matching-changes "$test_cmd" "test/" "docker/*" "buildspec.yml"
       - test_cmd="IGNORE_COVERAGE=- tox -e py27 -- test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $CPU_TAG_PY2 --py-version 2 --processor cpu --durations 10"
-      - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "docker/*" "buildspec.yml"
+      - execute-command-if-has-matching-changes "$test_cmd" "test/" "docker/*" "buildspec.yml"
       - test_cmd="IGNORE_COVERAGE=- tox -e py27 -- test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $GPU_TAG_PY2 --py-version 2 --processor gpu --durations 10"
-      - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "docker/*" "buildspec.yml"
+      - execute-command-if-has-matching-changes "$test_cmd" "test/" "docker/*" "buildspec.yml"
 
     finally:
       # shut down remote gpu instance
       - cleanup-gpu-instances
       - cleanup-key-pairs
 
       # remove ecr image
-      - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_TAG_PY2
-      - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_TAG_PY2
-      - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_TAG_PY3
-      - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_TAG_PY3
+      - delete_cmd="aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_TAG_PY2"
+      - execute-command-if-has-matching-changes "$delete_cmd" "test/" "docker/*" "buildspec.yml"
+      - delete_cmd="aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_TAG_PY2"
+      - execute-command-if-has-matching-changes "$delete_cmd" "test/" "docker/*" "buildspec.yml"
+      - delete_cmd="aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_TAG_PY3"
+      - execute-command-if-has-matching-changes "$delete_cmd" "test/" "docker/*" "buildspec.yml"
+      - delete_cmd="aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_TAG_PY3"
+      - execute-command-if-has-matching-changes "$delete_cmd" "test/" "docker/*" "buildspec.yml"

docker/1.15.2/py2/Dockerfile.cpu

@@ -22,7 +22,6 @@ ENV LC_ALL=C.UTF-8
 ENV SAGEMAKER_TRAINING_MODULE=sagemaker_tensorflow_container.training:main
 
 # Define framework-related package sources
-ARG FRAMEWORK_SUPPORT_INSTALLABLE=sagemaker_tensorflow_training*.tar.gz
 ARG TF_URL=https://tensorflow-aws.s3-us-west-2.amazonaws.com/1.15.2/AmazonLinux/cpu/final/tensorflow-1.15.2-cp27-cp27mu-manylinux2010_x86_64.whl
 
 RUN apt-get update \
@@ -80,8 +79,6 @@ RUN apt-get update \
     python \
     python-pip
 
-COPY $FRAMEWORK_SUPPORT_INSTALLABLE .
-
 RUN pip --no-cache-dir install --upgrade \
     pip \
     setuptools
@@ -103,13 +100,12 @@ RUN pip install --no-cache-dir -U \
     mpi4py==3.0.2 \
     "cryptography>=2.3" \
     "sagemaker-tensorflow>=1.15,<1.16" \
+    "sagemaker-tensorflow-training>=2,<3" \
     # Let's install TensorFlow separately in the end to avoid the library version to be overwritten
  && pip install --force-reinstall --no-cache-dir -U \
     ${TF_URL} \
  && pip install --no-cache-dir -U \
-    $FRAMEWORK_SUPPORT_INSTALLABLE \
-    awscli\
- && rm -f $FRAMEWORK_SUPPORT_INSTALLABLE \
+    awscli \
  && pip install --no-cache-dir -U \
     horovod==0.18.2
 

docker/1.15.2/py2/Dockerfile.gpu

@@ -19,7 +19,6 @@ ENV LC_ALL=C.UTF-8
 ENV SAGEMAKER_TRAINING_MODULE=sagemaker_tensorflow_container.training:main
 
 # Define framework-related package sources
-ARG FRAMEWORK_SUPPORT_INSTALLABLE=sagemaker_tensorflow_training*.tar.gz
 ARG TF_URL=https://tensorflow-aws.s3-us-west-2.amazonaws.com/1.15.2/AmazonLinux/gpu/final/tensorflow_gpu-1.15.2-cp27-cp27mu-manylinux2010_x86_64.whl
 
 RUN apt-get update \
@@ -120,8 +119,6 @@ RUN pip --no-cache-dir install --upgrade \
 # Some TF tools expect a "python" binary
 RUN ln -s $(which python) /usr/local/bin/python
 
-COPY $FRAMEWORK_SUPPORT_INSTALLABLE .
-
 RUN pip install --no-cache-dir -U \
     numpy==1.16.5 \
     scipy==1.2.2 \
@@ -136,13 +133,12 @@ RUN pip install --no-cache-dir -U \
     mpi4py==3.0.2 \
     "cryptography>=2.3" \
     "sagemaker-tensorflow>=1.15,<1.16" \
+    "sagemaker-tensorflow-training>=2,<3" \
     # Let's install TensorFlow separately in the end to avoid the library version to be overwritten
  && pip install --force-reinstall --no-cache-dir -U \
     ${TF_URL} \
  && pip install --no-cache-dir -U \
-    $FRAMEWORK_SUPPORT_INSTALLABLE \
-    awscli\
- && rm -f $FRAMEWORK_SUPPORT_INSTALLABLE
+    awscli
 
 # Install Horovod, temporarily using CUDA stubs
 RUN ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs \

docker/1.15.2/py3/Dockerfile.cpu

@@ -22,7 +22,6 @@ ENV LC_ALL=C.UTF-8
 ENV SAGEMAKER_TRAINING_MODULE=sagemaker_tensorflow_container.training:main
 
 # Define framework-related package sources
-ARG FRAMEWORK_SUPPORT_INSTALLABLE=sagemaker_tensorflow_training*.tar.gz
 ARG TF_URL=https://tensorflow-aws.s3-us-west-2.amazonaws.com/1.15.2/AmazonLinux/cpu/final/tensorflow-1.15.2-cp36-cp36m-manylinux2010_x86_64.whl
 
 RUN apt-get update \
@@ -78,8 +77,6 @@ RUN mkdir -p /root/.ssh/ \
 
 WORKDIR /
 
-COPY $FRAMEWORK_SUPPORT_INSTALLABLE .
-
 RUN pip3 --no-cache-dir install --upgrade \
     pip \
     setuptools
@@ -105,16 +102,15 @@ RUN pip install --no-cache-dir -U \
     mpi4py==3.0.2 \
     "cryptography>=2.3" \
     "sagemaker-tensorflow>=1.15,<1.16" \
+    "sagemaker-tensorflow-training>=2,<3" \
     # Let's install TensorFlow separately in the end to avoid
     # the library version to be overwritten
  && pip install --force-reinstall --no-cache-dir -U \
     ${TF_URL} \
  && pip install --force-reinstall --no-cache-dir -U \
     horovod==0.18.2 \
  && pip install --no-cache-dir -U \
-    $FRAMEWORK_SUPPORT_INSTALLABLE \
-    awscli\
- && rm -f $FRAMEWORK_SUPPORT_INSTALLABLE
+    awscli
 
 ADD https://raw.githubusercontent.com/aws/aws-deep-learning-containers-utils/master/deep_learning_container.py /usr/local/bin/deep_learning_container.py
 

docker/1.15.2/py3/Dockerfile.gpu

@@ -19,7 +19,6 @@ ENV LC_ALL=C.UTF-8
 ENV SAGEMAKER_TRAINING_MODULE=sagemaker_tensorflow_container.training:main
 
 # Define framework-related package sources
-ARG FRAMEWORK_SUPPORT_INSTALLABLE=sagemaker_tensorflow_training*.tar.gz
 ARG TF_URL=https://tensorflow-aws.s3-us-west-2.amazonaws.com/1.15.2/AmazonLinux/gpu/final/tensorflow_gpu-1.15.2-cp36-cp36m-manylinux2010_x86_64.whl
 
 RUN apt-get update \
@@ -123,8 +122,6 @@ RUN pip3 --no-cache-dir install --upgrade \
 RUN ln -s $(which python3) /usr/local/bin/python \
  && ln -s $(which pip3) /usr/bin/pip
 
-COPY $FRAMEWORK_SUPPORT_INSTALLABLE .
-
 RUN pip install --no-cache-dir -U \
     numpy==1.17.4 \
     scipy==1.2.2 \
@@ -142,14 +139,13 @@ RUN pip install --no-cache-dir -U \
     mpi4py==3.0.2 \
     "cryptography>=2.3" \
     "sagemaker-tensorflow>=1.15,<1.16" \
+    "sagemaker-tensorflow-training>=2,<3" \
     # Let's install TensorFlow separately in the end to avoid
     # the library version to be overwritten
  && pip install --force-reinstall --no-cache-dir -U \
     ${TF_URL} \
  && pip install --no-cache-dir -U \
-    $FRAMEWORK_SUPPORT_INSTALLABLE \
-    awscli\
- && rm -f $FRAMEWORK_SUPPORT_INSTALLABLE
+    awscli
 
 # Install Horovod, temporarily using CUDA stubs
 RUN ldconfig /usr/local/cuda-10.0/targets/x86_64-linux/lib/stubs \