Remove numpy pin from ci/docker/conda-python-spark.dockerfile and add it to tasks.yml

Author: AlenkaF

ci/docker/conda-python-spark.dockerfile

@@ -22,15 +22,10 @@ FROM ${repo}:${arch}-conda-python-${python}
 
 ARG jdk=8
 ARG maven=3.5
-# https://github.com/apache/arrow/issues/33697
-# numpy version pin should be removed with new apache spark release
-# that includes https://github.com/apache/spark/pull/37817
-ARG numpy=1.23
 
 RUN mamba install -q -y \
         openjdk=${jdk} \
         maven=${maven} \
-        numpy=${numpy} \
         pandas && \
     mamba clean --all
 

dev/tasks/tasks.yml

@@ -1587,9 +1587,9 @@ tasks:
       image: conda-python-hdfs
 {% endfor %}
 
-{% for python_version, spark_version, test_pyarrow_only in [("3.7", "v3.1.2", "false"),
-                                                            ("3.8", "v3.2.0", "false"),
-                                                            ("3.9", "master", "false")] %}
+{% for python_version, spark_version, test_pyarrow_only, numpy_version in [("3.7", "v3.1.2", "false", "latest"),
+                                                                           ("3.8", "v3.2.0", "false", "1.23"),
+                                                                           ("3.9", "master", "false", "latest")] %}
   test-conda-python-{{ python_version }}-spark-{{ spark_version }}:
     ci: github
     template: docker-tests/github.linux.yml
@@ -1598,6 +1598,7 @@ tasks:
         PYTHON: "{{ python_version }}"
         SPARK: "{{ spark_version }}"
         TEST_PYARROW_ONLY: "{{ test_pyarrow_only }}"
+        NUMPY: "{{ numpy_version }}"
       # use the branch-3.0 of spark, so prevent reusing any layers
       flags: --no-leaf-cache
       image: conda-python-spark