Browse Source
* Modify docker-compose and dockerfile 1. Add python3 in Dockerfile 2. Modify startup.sh file 3. Modify docker-compose.yml and add docker-stack.yml 4. Modify dolphinscheduler_env.sh * Add logback-xx.xml and Modify Dockerfilepull/2/head
liwenhe1993
5 years ago
committed by
GitHub
10 changed files with 512 additions and 108 deletions
@ -0,0 +1,230 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
version: "3.4" |
||||
|
||||
services: |
||||
|
||||
dolphinscheduler-postgresql: |
||||
image: bitnami/postgresql:latest |
||||
ports: |
||||
- 5432:5432 |
||||
environment: |
||||
TZ: Asia/Shanghai |
||||
POSTGRESQL_USERNAME: root |
||||
POSTGRESQL_PASSWORD: root |
||||
POSTGRESQL_DATABASE: dolphinscheduler |
||||
volumes: |
||||
- dolphinscheduler-postgresql:/bitnami/postgresql |
||||
networks: |
||||
- dolphinscheduler |
||||
deploy: |
||||
mode: replicated |
||||
replicas: 1 |
||||
|
||||
dolphinscheduler-zookeeper: |
||||
image: bitnami/zookeeper:latest |
||||
ports: |
||||
- 2181:2181 |
||||
environment: |
||||
TZ: Asia/Shanghai |
||||
ALLOW_ANONYMOUS_LOGIN: "yes" |
||||
volumes: |
||||
- dolphinscheduler-zookeeper:/bitnami/zookeeper |
||||
networks: |
||||
- dolphinscheduler |
||||
deploy: |
||||
mode: replicated |
||||
replicas: 1 |
||||
|
||||
dolphinscheduler-api: |
||||
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev |
||||
command: ["api-server"] |
||||
ports: |
||||
- 12345:12345 |
||||
environment: |
||||
TZ: Asia/Shanghai |
||||
POSTGRESQL_HOST: dolphinscheduler-postgresql |
||||
POSTGRESQL_PORT: 5432 |
||||
POSTGRESQL_USERNAME: root |
||||
POSTGRESQL_PASSWORD: root |
||||
POSTGRESQL_DATABASE: dolphinscheduler |
||||
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 |
||||
healthcheck: |
||||
test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] |
||||
interval: 30 |
||||
timeout: 5s |
||||
retries: 3 |
||||
start_period: 30s |
||||
volumes: |
||||
- dolphinscheduler-api:/opt/dolphinscheduler/logs |
||||
networks: |
||||
- dolphinscheduler |
||||
deploy: |
||||
mode: replicated |
||||
replicas: 1 |
||||
|
||||
dolphinscheduler-frontend: |
||||
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev |
||||
command: ["frontend"] |
||||
ports: |
||||
- 8888:8888 |
||||
environment: |
||||
TZ: Asia/Shanghai |
||||
FRONTEND_API_SERVER_HOST: dolphinscheduler-api |
||||
FRONTEND_API_SERVER_PORT: 12345 |
||||
healthcheck: |
||||
test: ["CMD", "nc", "-z", "localhost", "8888"] |
||||
interval: 30 |
||||
timeout: 5s |
||||
retries: 3 |
||||
start_period: 30s |
||||
volumes: |
||||
- dolphinscheduler-frontend:/var/log/nginx |
||||
networks: |
||||
- dolphinscheduler |
||||
deploy: |
||||
mode: replicated |
||||
replicas: 1 |
||||
|
||||
dolphinscheduler-alert: |
||||
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev |
||||
command: ["alert-server"] |
||||
environment: |
||||
TZ: Asia/Shanghai |
||||
XLS_FILE_PATH: "/tmp/xls" |
||||
MAIL_SERVER_HOST: "" |
||||
MAIL_SERVER_PORT: "" |
||||
MAIL_SENDER: "" |
||||
MAIL_USER: "" |
||||
MAIL_PASSWD: "" |
||||
MAIL_SMTP_STARTTLS_ENABLE: "false" |
||||
MAIL_SMTP_SSL_ENABLE: "false" |
||||
MAIL_SMTP_SSL_TRUST: "" |
||||
ENTERPRISE_WECHAT_ENABLE: "false" |
||||
ENTERPRISE_WECHAT_CORP_ID: "" |
||||
ENTERPRISE_WECHAT_SECRET: "" |
||||
ENTERPRISE_WECHAT_AGENT_ID: "" |
||||
ENTERPRISE_WECHAT_USERS: "" |
||||
POSTGRESQL_HOST: dolphinscheduler-postgresql |
||||
POSTGRESQL_PORT: 5432 |
||||
POSTGRESQL_USERNAME: root |
||||
POSTGRESQL_PASSWORD: root |
||||
POSTGRESQL_DATABASE: dolphinscheduler |
||||
healthcheck: |
||||
test: ["CMD", "/root/checkpoint.sh", "AlertServer"] |
||||
interval: 30 |
||||
timeout: 5s |
||||
retries: 3 |
||||
start_period: 30s |
||||
volumes: |
||||
- dolphinscheduler-alert:/opt/dolphinscheduler/logs |
||||
networks: |
||||
- dolphinscheduler |
||||
deploy: |
||||
mode: replicated |
||||
replicas: 1 |
||||
|
||||
dolphinscheduler-master: |
||||
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev |
||||
command: ["master-server"] |
||||
ports: |
||||
- 5678:5678 |
||||
environment: |
||||
TZ: Asia/Shanghai |
||||
MASTER_EXEC_THREADS: "100" |
||||
MASTER_EXEC_TASK_NUM: "20" |
||||
MASTER_HEARTBEAT_INTERVAL: "10" |
||||
MASTER_TASK_COMMIT_RETRYTIMES: "5" |
||||
MASTER_TASK_COMMIT_INTERVAL: "1000" |
||||
MASTER_MAX_CPULOAD_AVG: "100" |
||||
MASTER_RESERVED_MEMORY: "0.1" |
||||
POSTGRESQL_HOST: dolphinscheduler-postgresql |
||||
POSTGRESQL_PORT: 5432 |
||||
POSTGRESQL_USERNAME: root |
||||
POSTGRESQL_PASSWORD: root |
||||
POSTGRESQL_DATABASE: dolphinscheduler |
||||
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 |
||||
healthcheck: |
||||
test: ["CMD", "/root/checkpoint.sh", "MasterServer"] |
||||
interval: 30 |
||||
timeout: 5s |
||||
retries: 3 |
||||
start_period: 30s |
||||
volumes: |
||||
- dolphinscheduler-master:/opt/dolphinscheduler/logs |
||||
networks: |
||||
- dolphinscheduler |
||||
deploy: |
||||
mode: replicated |
||||
replicas: 1 |
||||
|
||||
dolphinscheduler-worker: |
||||
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev |
||||
command: ["worker-server"] |
||||
ports: |
||||
- 1234:1234 |
||||
- 50051:50051 |
||||
environment: |
||||
TZ: Asia/Shanghai |
||||
WORKER_EXEC_THREADS: "100" |
||||
WORKER_HEARTBEAT_INTERVAL: "10" |
||||
WORKER_FETCH_TASK_NUM: "3" |
||||
WORKER_MAX_CPULOAD_AVG: "100" |
||||
WORKER_RESERVED_MEMORY: "0.1" |
||||
WORKER_GROUP: "default" |
||||
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" |
||||
POSTGRESQL_HOST: dolphinscheduler-postgresql |
||||
POSTGRESQL_PORT: 5432 |
||||
POSTGRESQL_USERNAME: root |
||||
POSTGRESQL_PASSWORD: root |
||||
POSTGRESQL_DATABASE: dolphinscheduler |
||||
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 |
||||
healthcheck: |
||||
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] |
||||
interval: 30 |
||||
timeout: 5s |
||||
retries: 3 |
||||
start_period: 30s |
||||
volumes: |
||||
- dolphinscheduler-worker-data:/tmp/dolphinscheduler |
||||
- dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs |
||||
configs: |
||||
- source: dolphinscheduler-worker-task-env |
||||
target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh |
||||
networks: |
||||
- dolphinscheduler |
||||
deploy: |
||||
mode: replicated |
||||
replicas: 1 |
||||
|
||||
networks: |
||||
dolphinscheduler: |
||||
driver: overlay |
||||
|
||||
volumes: |
||||
dolphinscheduler-postgresql: |
||||
dolphinscheduler-zookeeper: |
||||
dolphinscheduler-api: |
||||
dolphinscheduler-frontend: |
||||
dolphinscheduler-alert: |
||||
dolphinscheduler-master: |
||||
dolphinscheduler-worker-data: |
||||
dolphinscheduler-worker-logs: |
||||
|
||||
configs: |
||||
dolphinscheduler-worker-task-env: |
||||
file: ./dolphinscheduler_env.sh |
||||
@ -0,0 +1,52 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
|
||||
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html --> |
||||
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" --> |
||||
|
||||
<property name="log.base" value="logs"/> |
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
|
||||
<appender name="ALERTLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||
<file>${log.base}/dolphinscheduler-alert.log</file> |
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||
<fileNamePattern>${log.base}/dolphinscheduler-alert.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||
<maxHistory>20</maxHistory> |
||||
<maxFileSize>64MB</maxFileSize> |
||||
</rollingPolicy> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
|
||||
<root level="INFO"> |
||||
<appender-ref ref="STDOUT"/> |
||||
<appender-ref ref="APILOGFILE"/> |
||||
</root> |
||||
|
||||
</configuration> |
||||
@ -0,0 +1,62 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
|
||||
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html --> |
||||
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" --> |
||||
|
||||
<property name="log.base" value="logs"/> |
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
|
||||
<!-- api server logback config start --> |
||||
<appender name="APILOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||
<file>${log.base}/dolphinscheduler-api-server.log</file> |
||||
<filter class="ch.qos.logback.classic.filter.ThresholdFilter"> |
||||
<level>INFO</level> |
||||
</filter> |
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||
<fileNamePattern>${log.base}/dolphinscheduler-api-server.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||
<maxHistory>168</maxHistory> |
||||
<maxFileSize>64MB</maxFileSize> |
||||
</rollingPolicy> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
<!-- api server logback config end --> |
||||
|
||||
<logger name="org.apache.zookeeper" level="WARN"/> |
||||
<logger name="org.apache.hbase" level="WARN"/> |
||||
<logger name="org.apache.hadoop" level="WARN"/> |
||||
|
||||
|
||||
<root level="INFO"> |
||||
<appender-ref ref="STDOUT"/> |
||||
<appender-ref ref="APILOGFILE"/> |
||||
</root> |
||||
|
||||
</configuration> |
||||
@ -0,0 +1,82 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
|
||||
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html --> |
||||
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" --> |
||||
|
||||
<property name="log.base" value="logs"/> |
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
|
||||
<conversionRule conversionWord="messsage" |
||||
converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/> |
||||
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender"> |
||||
<filter class="ch.qos.logback.classic.filter.ThresholdFilter"> |
||||
<level>INFO</level> |
||||
</filter> |
||||
<filter class="org.apache.dolphinscheduler.server.log.TaskLogFilter"/> |
||||
<Discriminator class="org.apache.dolphinscheduler.server.log.TaskLogDiscriminator"> |
||||
<key>taskAppId</key> |
||||
<logBase>${log.base}</logBase> |
||||
</Discriminator> |
||||
<sift> |
||||
<appender name="FILE-${taskAppId}" class="ch.qos.logback.core.FileAppender"> |
||||
<file>${log.base}/${taskAppId}.log</file> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
<append>true</append> |
||||
</appender> |
||||
</sift> |
||||
</appender> |
||||
<!-- master server logback config start --> |
||||
<appender name="MASTERLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||
<file>${log.base}/dolphinscheduler-master.log</file> |
||||
<!--<filter class="org.apache.dolphinscheduler.server.log.MasterLogFilter"> |
||||
<level>INFO</level> |
||||
</filter>--> |
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||
<fileNamePattern>${log.base}/dolphinscheduler-master.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||
<maxHistory>168</maxHistory> |
||||
<maxFileSize>200MB</maxFileSize> |
||||
</rollingPolicy> |
||||
<encoder> |
||||
<pattern> |
||||
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||
</pattern> |
||||
<charset>UTF-8</charset> |
||||
</encoder> |
||||
</appender> |
||||
<!-- master server logback config end --> |
||||
|
||||
<root level="INFO"> |
||||
<appender-ref ref="STDOUT"/> |
||||
<appender-ref ref="TASKLOGFILE"/> |
||||
<appender-ref ref="MASTERLOGFILE"/> |
||||
</root> |
||||
|
||||
</configuration> |
||||
Loading…
Reference in new issue