[Feature-7993][E2E] Replace HDFS image with MinIO S3 image (#8066)

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties

@@ -19,7 +19,7 @@
 data.basedir.path=/tmp/dolphinscheduler
 
 # resource storage type: HDFS, S3, NONE
-resource.storage.type=HDFS
+resource.storage.type=S3
 
 # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
 resource.upload.path=/dolphinscheduler
@@ -46,16 +46,16 @@ kerberos.expire.time=2
 hdfs.root.user=hdfs
 
 # if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
-fs.defaultFS=hdfs://hdfs:8020
+fs.defaultFS=s3a://dolphinscheduler
 
 # if resource.storage.type=S3, s3 endpoint
-fs.s3a.endpoint=http://192.168.xx.xx:9010
+fs.s3a.endpoint=http://10.1.0.1:9000
 
 # if resource.storage.type=S3, s3 access key
-fs.s3a.access.key=A3DXS30FO22544RE
+fs.s3a.access.key=accessKey123
 
 # if resource.storage.type=S3, s3 secret key
-fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
+fs.s3a.secret.key=secretKey123
 
 # resourcemanager port, the default value is 8088 if not specified
 resource.manager.httpaddress.port=8088

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml

@@ -35,22 +35,51 @@ services:
     volumes:
       - ./common.properties:/opt/dolphinscheduler/conf/common.properties
     depends_on:
-      hdfs:
+      s3:
         condition: service_healthy
-  hdfs:
+      mc:
-    image: mdouchement/hdfs:latest
+        condition: service_healthy
-    hostname: hdfs
+  s3:
+    image: minio/minio:latest
+    hostname: s3
     tty: true
     stdin_open: true
-    expose:
+    command: server /data --console-address ":9001"
-      - 8020
+    ports:
+      - 9000:9000
     networks:
       - e2e
+    environment:
+      MINIO_ROOT_USER: accessKey123
+      MINIO_ROOT_PASSWORD: secretKey123
     healthcheck:
-      test: [ "CMD", "curl", "http://localhost:50070" ]
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 5s
       timeout: 120s
       retries: 120
-
+  mc:
+    image: minio/mc:latest
+    entrypoint: ""
+    hostname: mc
+    tty: true
+    stdin_open: true
+    networks:
+      - e2e
+    command: bash -c '
+      mc alias set s3 http://s3:9000 accessKey123 secretKey123
+      && mc mb s3/dolphinscheduler && tail -f /dev/null'
+    healthcheck:
+      test: [ "CMD", "echo", "1" ]
+      interval: 5s
+      timeout: 120s
+      retries: 120
+    depends_on:
+      s3:
+        condition: service_healthy
 networks:
   e2e:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 10.1.0.0/24
+          gateway: 10.1.0.1