From a8befd5db30a778cfc8223b2b21ec4747656f9f0 Mon Sep 17 00:00:00 2001 From: Weichen Xu Date: Tue, 28 Feb 2023 19:16:48 +0800 Subject: [PATCH 1/6] init Signed-off-by: Weichen Xu --- docker/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f358cd69..0f17e3a3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -43,7 +43,8 @@ RUN /petastorm_venv3.7/bin/pip3.7 install --no-cache scikit-build RUN /petastorm_venv3.7/bin/pip3.7 install --no-cache -e /petastorm/[test,tf,torch,docs,opencv] --only-binary pyarrow --only-binary opencv-python RUN /petastorm_venv3.7/bin/pip3.7 uninstall -y petastorm # To avoid some version incompatibilities, we pin these libraries to versions that known to work together -RUN /petastorm_venv3.7/bin/pip3.7 install -U pyarrow==3.0.0 numpy==1.19.1 tensorflow==2.1.0 pyspark==3.0.0 +RUN /petastorm_venv3.7/bin/pip3.7 install -U pyarrow==3.0.0 numpy==1.19.1 tensorflow==2.1.0 +RUN /petastorm_venv3.7/bin/pip3.7 install -U https://dist.apache.org/repos/dist/dev/spark/v3.4.0-rc1-bin/pyspark-3.4.0.tar.gz # Otherwise we might have trouble with loading of libGL.so.1 RUN /petastorm_venv3.7/bin/pip3.7 install opencv-python-headless @@ -58,7 +59,8 @@ RUN /petastorm_venv3.9/bin/pip3.9 install --no-cache scikit-build RUN /petastorm_venv3.9/bin/pip3.9 install --no-cache -e /petastorm/[test,tf,torch,docs,opencv] --only-binary pyarrow --only-binary opencv-python RUN /petastorm_venv3.9/bin/pip3.9 uninstall -y petastorm # To avoid some version incompatibilities, we pin these libraries to versions that known to work together -RUN /petastorm_venv3.9/bin/pip3.9 install -U pyarrow==3.0.0 numpy==1.19.3 tensorflow==2.5.0 pyspark==3.0.0 +RUN /petastorm_venv3.9/bin/pip3.9 install -U pyarrow==3.0.0 numpy==1.19.3 tensorflow==2.5.0 +RUN /petastorm_venv3.7/bin/pip3.7 install -U https://dist.apache.org/repos/dist/dev/spark/v3.4.0-rc1-bin/pyspark-3.4.0.tar.gz # Otherwise we might have trouble with loading of libGL.so.1 RUN /petastorm_venv3.9/bin/pip3.9 install opencv-python-headless From 7fc118dc8ea4d4b12524e05ab7f8aa491c5a17bb Mon Sep 17 00:00:00 2001 From: Weichen Xu Date: Tue, 28 Feb 2023 19:56:46 +0800 Subject: [PATCH 2/6] update Signed-off-by: Weichen Xu --- .github/workflows/unittest.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 3bac1a99..87f2e3f6 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -26,39 +26,32 @@ jobs: matrix: config: [pyspark-2.4, tf-1.15, pyarrow-3.0, pyarrow-4.0, latest] include: - - config: pyspark-2.4 - PYARROW_VERSION: "2.0.0" - NUMPY_VERSION: "1.19.1" - TF_VERSION: "1.15.5" - PYSPARK_VERSION: "2.4.4" - ARROW_PRE_0_15_IPC_FORMAT: 1 - PY: "3.7" - config: tf-1.15 PYARROW_VERSION: "2.0.0" NUMPY_VERSION: "1.19.1" TF_VERSION: "1.15.5" - PYSPARK_VERSION: "3.0.0" + PYSPARK_VERSION: "3.4.0" ARROW_PRE_0_15_IPC_FORMAT: 0 PY: "3.7" - config: pyarrow-3.0 PYARROW_VERSION: "3.0.0" NUMPY_VERSION: "1.19.1" TF_VERSION: "2.5.0" - PYSPARK_VERSION: "3.0.0" + PYSPARK_VERSION: "3.4.0" ARROW_PRE_0_15_IPC_FORMAT: 0 PY: "3.7" - config: pyarrow-4.0 PYARROW_VERSION: "4.0.0" NUMPY_VERSION: "1.19.1" TF_VERSION: "2.5.0" - PYSPARK_VERSION: "3.0.0" + PYSPARK_VERSION: "3.4.0" ARROW_PRE_0_15_IPC_FORMAT: 0 PY: "3.7" - config: latest PYARROW_VERSION: "6.0.1" NUMPY_VERSION: "1.21.5" TF_VERSION: "2.8.0" - PYSPARK_VERSION: "3.0.0" + PYSPARK_VERSION: "3.4.0" ARROW_PRE_0_15_IPC_FORMAT: "0" PY: "3.9" @@ -91,7 +84,8 @@ jobs: $RUN pip install -U pip setuptools $RUN pip install -e /petastorm/[test,tf,torch,docs,opencv] $RUN pip install --upgrade numpy==$NUMPY_VERSION - $RUN pip install -U pyarrow==${PYARROW_VERSION} tensorflow==${TF_VERSION} pyspark==${PYSPARK_VERSION} + $RUN pip install -U pyarrow==${PYARROW_VERSION} tensorflow==${TF_VERSION} + $RUN pip install -U https://dist.apache.org/repos/dist/dev/spark/v3.4.0-rc1-bin/pyspark-3.4.0.tar.gz $RUN pip list $RUN mypy petastorm $RUN flake8 . --count --show-source --statistics From 4bd450b146341e6e8e24298cba626be9e382eaa9 Mon Sep 17 00:00:00 2001 From: Weichen Xu Date: Tue, 28 Feb 2023 20:01:07 +0800 Subject: [PATCH 3/6] update Signed-off-by: Weichen Xu --- .github/workflows/unittest.yml | 2 +- docker/Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 87f2e3f6..d3d58603 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -85,7 +85,7 @@ jobs: $RUN pip install -e /petastorm/[test,tf,torch,docs,opencv] $RUN pip install --upgrade numpy==$NUMPY_VERSION $RUN pip install -U pyarrow==${PYARROW_VERSION} tensorflow==${TF_VERSION} - $RUN pip install -U https://dist.apache.org/repos/dist/dev/spark/v3.4.0-rc1-bin/pyspark-3.4.0.tar.gz + $RUN pip install https://dist.apache.org/repos/dist/dev/spark/v3.4.0-rc1-bin/pyspark-3.4.0.tar.gz $RUN pip list $RUN mypy petastorm $RUN flake8 . --count --show-source --statistics diff --git a/docker/Dockerfile b/docker/Dockerfile index 0f17e3a3..5045c547 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -44,7 +44,7 @@ RUN /petastorm_venv3.7/bin/pip3.7 install --no-cache -e /petastorm/[test,tf,torc RUN /petastorm_venv3.7/bin/pip3.7 uninstall -y petastorm # To avoid some version incompatibilities, we pin these libraries to versions that known to work together RUN /petastorm_venv3.7/bin/pip3.7 install -U pyarrow==3.0.0 numpy==1.19.1 tensorflow==2.1.0 -RUN /petastorm_venv3.7/bin/pip3.7 install -U https://dist.apache.org/repos/dist/dev/spark/v3.4.0-rc1-bin/pyspark-3.4.0.tar.gz +RUN /petastorm_venv3.7/bin/pip3.7 install https://dist.apache.org/repos/dist/dev/spark/v3.4.0-rc1-bin/pyspark-3.4.0.tar.gz # Otherwise we might have trouble with loading of libGL.so.1 RUN /petastorm_venv3.7/bin/pip3.7 install opencv-python-headless @@ -60,7 +60,7 @@ RUN /petastorm_venv3.9/bin/pip3.9 install --no-cache -e /petastorm/[test,tf,torc RUN /petastorm_venv3.9/bin/pip3.9 uninstall -y petastorm # To avoid some version incompatibilities, we pin these libraries to versions that known to work together RUN /petastorm_venv3.9/bin/pip3.9 install -U pyarrow==3.0.0 numpy==1.19.3 tensorflow==2.5.0 -RUN /petastorm_venv3.7/bin/pip3.7 install -U https://dist.apache.org/repos/dist/dev/spark/v3.4.0-rc1-bin/pyspark-3.4.0.tar.gz +RUN /petastorm_venv3.7/bin/pip3.7 install https://dist.apache.org/repos/dist/dev/spark/v3.4.0-rc1-bin/pyspark-3.4.0.tar.gz # Otherwise we might have trouble with loading of libGL.so.1 RUN /petastorm_venv3.9/bin/pip3.9 install opencv-python-headless From a07c2553f46cf823f8b2356764a8d0b10ecf9c94 Mon Sep 17 00:00:00 2001 From: Weichen Xu Date: Tue, 28 Feb 2023 20:03:23 +0800 Subject: [PATCH 4/6] update Signed-off-by: Weichen Xu --- .github/workflows/unittest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index d3d58603..75b36b7b 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - config: [pyspark-2.4, tf-1.15, pyarrow-3.0, pyarrow-4.0, latest] + config: [tf-1.15, pyarrow-3.0, pyarrow-4.0, latest] include: - config: tf-1.15 PYARROW_VERSION: "2.0.0" From 38e416cdc239f07a20602665af30dd0d235500eb Mon Sep 17 00:00:00 2001 From: Weichen Xu Date: Tue, 28 Feb 2023 20:22:48 +0800 Subject: [PATCH 5/6] update Signed-off-by: Weichen Xu --- .github/workflows/unittest.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 75b36b7b..e895c428 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -32,21 +32,21 @@ jobs: TF_VERSION: "1.15.5" PYSPARK_VERSION: "3.4.0" ARROW_PRE_0_15_IPC_FORMAT: 0 - PY: "3.7" + PY: "3.9" - config: pyarrow-3.0 PYARROW_VERSION: "3.0.0" NUMPY_VERSION: "1.19.1" TF_VERSION: "2.5.0" PYSPARK_VERSION: "3.4.0" ARROW_PRE_0_15_IPC_FORMAT: 0 - PY: "3.7" + PY: "3.9" - config: pyarrow-4.0 PYARROW_VERSION: "4.0.0" NUMPY_VERSION: "1.19.1" TF_VERSION: "2.5.0" PYSPARK_VERSION: "3.4.0" ARROW_PRE_0_15_IPC_FORMAT: 0 - PY: "3.7" + PY: "3.9" - config: latest PYARROW_VERSION: "6.0.1" NUMPY_VERSION: "1.21.5" From 2e681387e0001ac759f2e289afe23cec1138d9eb Mon Sep 17 00:00:00 2001 From: Weichen Xu Date: Tue, 28 Feb 2023 20:35:17 +0800 Subject: [PATCH 6/6] update Signed-off-by: Weichen Xu --- .github/workflows/unittest.yml | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index e895c428..f9049627 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -24,29 +24,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - config: [tf-1.15, pyarrow-3.0, pyarrow-4.0, latest] + config: [latest] include: - - config: tf-1.15 - PYARROW_VERSION: "2.0.0" - NUMPY_VERSION: "1.19.1" - TF_VERSION: "1.15.5" - PYSPARK_VERSION: "3.4.0" - ARROW_PRE_0_15_IPC_FORMAT: 0 - PY: "3.9" - - config: pyarrow-3.0 - PYARROW_VERSION: "3.0.0" - NUMPY_VERSION: "1.19.1" - TF_VERSION: "2.5.0" - PYSPARK_VERSION: "3.4.0" - ARROW_PRE_0_15_IPC_FORMAT: 0 - PY: "3.9" - - config: pyarrow-4.0 - PYARROW_VERSION: "4.0.0" - NUMPY_VERSION: "1.19.1" - TF_VERSION: "2.5.0" - PYSPARK_VERSION: "3.4.0" - ARROW_PRE_0_15_IPC_FORMAT: 0 - PY: "3.9" - config: latest PYARROW_VERSION: "6.0.1" NUMPY_VERSION: "1.21.5"