samz406
5 years ago
committed by
GitHub
31 changed files with 738 additions and 162 deletions
@ -0,0 +1,230 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, software |
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
# See the License for the specific language governing permissions and |
||||||
|
# limitations under the License. |
||||||
|
|
||||||
|
version: "3.4" |
||||||
|
|
||||||
|
services: |
||||||
|
|
||||||
|
dolphinscheduler-postgresql: |
||||||
|
image: bitnami/postgresql:latest |
||||||
|
ports: |
||||||
|
- 5432:5432 |
||||||
|
environment: |
||||||
|
TZ: Asia/Shanghai |
||||||
|
POSTGRESQL_USERNAME: root |
||||||
|
POSTGRESQL_PASSWORD: root |
||||||
|
POSTGRESQL_DATABASE: dolphinscheduler |
||||||
|
volumes: |
||||||
|
- dolphinscheduler-postgresql:/bitnami/postgresql |
||||||
|
networks: |
||||||
|
- dolphinscheduler |
||||||
|
deploy: |
||||||
|
mode: replicated |
||||||
|
replicas: 1 |
||||||
|
|
||||||
|
dolphinscheduler-zookeeper: |
||||||
|
image: bitnami/zookeeper:latest |
||||||
|
ports: |
||||||
|
- 2181:2181 |
||||||
|
environment: |
||||||
|
TZ: Asia/Shanghai |
||||||
|
ALLOW_ANONYMOUS_LOGIN: "yes" |
||||||
|
volumes: |
||||||
|
- dolphinscheduler-zookeeper:/bitnami/zookeeper |
||||||
|
networks: |
||||||
|
- dolphinscheduler |
||||||
|
deploy: |
||||||
|
mode: replicated |
||||||
|
replicas: 1 |
||||||
|
|
||||||
|
dolphinscheduler-api: |
||||||
|
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev |
||||||
|
command: ["api-server"] |
||||||
|
ports: |
||||||
|
- 12345:12345 |
||||||
|
environment: |
||||||
|
TZ: Asia/Shanghai |
||||||
|
POSTGRESQL_HOST: dolphinscheduler-postgresql |
||||||
|
POSTGRESQL_PORT: 5432 |
||||||
|
POSTGRESQL_USERNAME: root |
||||||
|
POSTGRESQL_PASSWORD: root |
||||||
|
POSTGRESQL_DATABASE: dolphinscheduler |
||||||
|
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 |
||||||
|
healthcheck: |
||||||
|
test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] |
||||||
|
interval: 30 |
||||||
|
timeout: 5s |
||||||
|
retries: 3 |
||||||
|
start_period: 30s |
||||||
|
volumes: |
||||||
|
- dolphinscheduler-api:/opt/dolphinscheduler/logs |
||||||
|
networks: |
||||||
|
- dolphinscheduler |
||||||
|
deploy: |
||||||
|
mode: replicated |
||||||
|
replicas: 1 |
||||||
|
|
||||||
|
dolphinscheduler-frontend: |
||||||
|
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev |
||||||
|
command: ["frontend"] |
||||||
|
ports: |
||||||
|
- 8888:8888 |
||||||
|
environment: |
||||||
|
TZ: Asia/Shanghai |
||||||
|
FRONTEND_API_SERVER_HOST: dolphinscheduler-api |
||||||
|
FRONTEND_API_SERVER_PORT: 12345 |
||||||
|
healthcheck: |
||||||
|
test: ["CMD", "nc", "-z", "localhost", "8888"] |
||||||
|
interval: 30 |
||||||
|
timeout: 5s |
||||||
|
retries: 3 |
||||||
|
start_period: 30s |
||||||
|
volumes: |
||||||
|
- dolphinscheduler-frontend:/var/log/nginx |
||||||
|
networks: |
||||||
|
- dolphinscheduler |
||||||
|
deploy: |
||||||
|
mode: replicated |
||||||
|
replicas: 1 |
||||||
|
|
||||||
|
dolphinscheduler-alert: |
||||||
|
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev |
||||||
|
command: ["alert-server"] |
||||||
|
environment: |
||||||
|
TZ: Asia/Shanghai |
||||||
|
XLS_FILE_PATH: "/tmp/xls" |
||||||
|
MAIL_SERVER_HOST: "" |
||||||
|
MAIL_SERVER_PORT: "" |
||||||
|
MAIL_SENDER: "" |
||||||
|
MAIL_USER: "" |
||||||
|
MAIL_PASSWD: "" |
||||||
|
MAIL_SMTP_STARTTLS_ENABLE: "false" |
||||||
|
MAIL_SMTP_SSL_ENABLE: "false" |
||||||
|
MAIL_SMTP_SSL_TRUST: "" |
||||||
|
ENTERPRISE_WECHAT_ENABLE: "false" |
||||||
|
ENTERPRISE_WECHAT_CORP_ID: "" |
||||||
|
ENTERPRISE_WECHAT_SECRET: "" |
||||||
|
ENTERPRISE_WECHAT_AGENT_ID: "" |
||||||
|
ENTERPRISE_WECHAT_USERS: "" |
||||||
|
POSTGRESQL_HOST: dolphinscheduler-postgresql |
||||||
|
POSTGRESQL_PORT: 5432 |
||||||
|
POSTGRESQL_USERNAME: root |
||||||
|
POSTGRESQL_PASSWORD: root |
||||||
|
POSTGRESQL_DATABASE: dolphinscheduler |
||||||
|
healthcheck: |
||||||
|
test: ["CMD", "/root/checkpoint.sh", "AlertServer"] |
||||||
|
interval: 30 |
||||||
|
timeout: 5s |
||||||
|
retries: 3 |
||||||
|
start_period: 30s |
||||||
|
volumes: |
||||||
|
- dolphinscheduler-alert:/opt/dolphinscheduler/logs |
||||||
|
networks: |
||||||
|
- dolphinscheduler |
||||||
|
deploy: |
||||||
|
mode: replicated |
||||||
|
replicas: 1 |
||||||
|
|
||||||
|
dolphinscheduler-master: |
||||||
|
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev |
||||||
|
command: ["master-server"] |
||||||
|
ports: |
||||||
|
- 5678:5678 |
||||||
|
environment: |
||||||
|
TZ: Asia/Shanghai |
||||||
|
MASTER_EXEC_THREADS: "100" |
||||||
|
MASTER_EXEC_TASK_NUM: "20" |
||||||
|
MASTER_HEARTBEAT_INTERVAL: "10" |
||||||
|
MASTER_TASK_COMMIT_RETRYTIMES: "5" |
||||||
|
MASTER_TASK_COMMIT_INTERVAL: "1000" |
||||||
|
MASTER_MAX_CPULOAD_AVG: "100" |
||||||
|
MASTER_RESERVED_MEMORY: "0.1" |
||||||
|
POSTGRESQL_HOST: dolphinscheduler-postgresql |
||||||
|
POSTGRESQL_PORT: 5432 |
||||||
|
POSTGRESQL_USERNAME: root |
||||||
|
POSTGRESQL_PASSWORD: root |
||||||
|
POSTGRESQL_DATABASE: dolphinscheduler |
||||||
|
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 |
||||||
|
healthcheck: |
||||||
|
test: ["CMD", "/root/checkpoint.sh", "MasterServer"] |
||||||
|
interval: 30 |
||||||
|
timeout: 5s |
||||||
|
retries: 3 |
||||||
|
start_period: 30s |
||||||
|
volumes: |
||||||
|
- dolphinscheduler-master:/opt/dolphinscheduler/logs |
||||||
|
networks: |
||||||
|
- dolphinscheduler |
||||||
|
deploy: |
||||||
|
mode: replicated |
||||||
|
replicas: 1 |
||||||
|
|
||||||
|
dolphinscheduler-worker: |
||||||
|
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev |
||||||
|
command: ["worker-server"] |
||||||
|
ports: |
||||||
|
- 1234:1234 |
||||||
|
- 50051:50051 |
||||||
|
environment: |
||||||
|
TZ: Asia/Shanghai |
||||||
|
WORKER_EXEC_THREADS: "100" |
||||||
|
WORKER_HEARTBEAT_INTERVAL: "10" |
||||||
|
WORKER_FETCH_TASK_NUM: "3" |
||||||
|
WORKER_MAX_CPULOAD_AVG: "100" |
||||||
|
WORKER_RESERVED_MEMORY: "0.1" |
||||||
|
WORKER_GROUP: "default" |
||||||
|
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" |
||||||
|
POSTGRESQL_HOST: dolphinscheduler-postgresql |
||||||
|
POSTGRESQL_PORT: 5432 |
||||||
|
POSTGRESQL_USERNAME: root |
||||||
|
POSTGRESQL_PASSWORD: root |
||||||
|
POSTGRESQL_DATABASE: dolphinscheduler |
||||||
|
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 |
||||||
|
healthcheck: |
||||||
|
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] |
||||||
|
interval: 30 |
||||||
|
timeout: 5s |
||||||
|
retries: 3 |
||||||
|
start_period: 30s |
||||||
|
volumes: |
||||||
|
- dolphinscheduler-worker-data:/tmp/dolphinscheduler |
||||||
|
- dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs |
||||||
|
configs: |
||||||
|
- source: dolphinscheduler-worker-task-env |
||||||
|
target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh |
||||||
|
networks: |
||||||
|
- dolphinscheduler |
||||||
|
deploy: |
||||||
|
mode: replicated |
||||||
|
replicas: 1 |
||||||
|
|
||||||
|
networks: |
||||||
|
dolphinscheduler: |
||||||
|
driver: overlay |
||||||
|
|
||||||
|
volumes: |
||||||
|
dolphinscheduler-postgresql: |
||||||
|
dolphinscheduler-zookeeper: |
||||||
|
dolphinscheduler-api: |
||||||
|
dolphinscheduler-frontend: |
||||||
|
dolphinscheduler-alert: |
||||||
|
dolphinscheduler-master: |
||||||
|
dolphinscheduler-worker-data: |
||||||
|
dolphinscheduler-worker-logs: |
||||||
|
|
||||||
|
configs: |
||||||
|
dolphinscheduler-worker-task-env: |
||||||
|
file: ./dolphinscheduler_env.sh |
||||||
@ -0,0 +1,52 @@ |
|||||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||||
|
<!-- |
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
~ contributor license agreements. See the NOTICE file distributed with |
||||||
|
~ this work for additional information regarding copyright ownership. |
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
~ (the "License"); you may not use this file except in compliance with |
||||||
|
~ the License. You may obtain a copy of the License at |
||||||
|
~ |
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
~ |
||||||
|
~ Unless required by applicable law or agreed to in writing, software |
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
~ See the License for the specific language governing permissions and |
||||||
|
~ limitations under the License. |
||||||
|
--> |
||||||
|
|
||||||
|
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html --> |
||||||
|
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" --> |
||||||
|
|
||||||
|
<property name="log.base" value="logs"/> |
||||||
|
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> |
||||||
|
<encoder> |
||||||
|
<pattern> |
||||||
|
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||||
|
</pattern> |
||||||
|
<charset>UTF-8</charset> |
||||||
|
</encoder> |
||||||
|
</appender> |
||||||
|
|
||||||
|
<appender name="ALERTLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||||
|
<file>${log.base}/dolphinscheduler-alert.log</file> |
||||||
|
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||||
|
<fileNamePattern>${log.base}/dolphinscheduler-alert.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||||
|
<maxHistory>20</maxHistory> |
||||||
|
<maxFileSize>64MB</maxFileSize> |
||||||
|
</rollingPolicy> |
||||||
|
<encoder> |
||||||
|
<pattern> |
||||||
|
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||||
|
</pattern> |
||||||
|
<charset>UTF-8</charset> |
||||||
|
</encoder> |
||||||
|
</appender> |
||||||
|
|
||||||
|
<root level="INFO"> |
||||||
|
<appender-ref ref="STDOUT"/> |
||||||
|
<appender-ref ref="APILOGFILE"/> |
||||||
|
</root> |
||||||
|
|
||||||
|
</configuration> |
||||||
@ -0,0 +1,62 @@ |
|||||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||||
|
<!-- |
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
~ contributor license agreements. See the NOTICE file distributed with |
||||||
|
~ this work for additional information regarding copyright ownership. |
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
~ (the "License"); you may not use this file except in compliance with |
||||||
|
~ the License. You may obtain a copy of the License at |
||||||
|
~ |
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
~ |
||||||
|
~ Unless required by applicable law or agreed to in writing, software |
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
~ See the License for the specific language governing permissions and |
||||||
|
~ limitations under the License. |
||||||
|
--> |
||||||
|
|
||||||
|
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html --> |
||||||
|
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" --> |
||||||
|
|
||||||
|
<property name="log.base" value="logs"/> |
||||||
|
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> |
||||||
|
<encoder> |
||||||
|
<pattern> |
||||||
|
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||||
|
</pattern> |
||||||
|
<charset>UTF-8</charset> |
||||||
|
</encoder> |
||||||
|
</appender> |
||||||
|
|
||||||
|
<!-- api server logback config start --> |
||||||
|
<appender name="APILOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||||
|
<file>${log.base}/dolphinscheduler-api-server.log</file> |
||||||
|
<filter class="ch.qos.logback.classic.filter.ThresholdFilter"> |
||||||
|
<level>INFO</level> |
||||||
|
</filter> |
||||||
|
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||||
|
<fileNamePattern>${log.base}/dolphinscheduler-api-server.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||||
|
<maxHistory>168</maxHistory> |
||||||
|
<maxFileSize>64MB</maxFileSize> |
||||||
|
</rollingPolicy> |
||||||
|
<encoder> |
||||||
|
<pattern> |
||||||
|
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||||
|
</pattern> |
||||||
|
<charset>UTF-8</charset> |
||||||
|
</encoder> |
||||||
|
</appender> |
||||||
|
<!-- api server logback config end --> |
||||||
|
|
||||||
|
<logger name="org.apache.zookeeper" level="WARN"/> |
||||||
|
<logger name="org.apache.hbase" level="WARN"/> |
||||||
|
<logger name="org.apache.hadoop" level="WARN"/> |
||||||
|
|
||||||
|
|
||||||
|
<root level="INFO"> |
||||||
|
<appender-ref ref="STDOUT"/> |
||||||
|
<appender-ref ref="APILOGFILE"/> |
||||||
|
</root> |
||||||
|
|
||||||
|
</configuration> |
||||||
@ -0,0 +1,82 @@ |
|||||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||||
|
<!-- |
||||||
|
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
~ contributor license agreements. See the NOTICE file distributed with |
||||||
|
~ this work for additional information regarding copyright ownership. |
||||||
|
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
~ (the "License"); you may not use this file except in compliance with |
||||||
|
~ the License. You may obtain a copy of the License at |
||||||
|
~ |
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
~ |
||||||
|
~ Unless required by applicable law or agreed to in writing, software |
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
~ See the License for the specific language governing permissions and |
||||||
|
~ limitations under the License. |
||||||
|
--> |
||||||
|
|
||||||
|
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html --> |
||||||
|
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" --> |
||||||
|
|
||||||
|
<property name="log.base" value="logs"/> |
||||||
|
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> |
||||||
|
<encoder> |
||||||
|
<pattern> |
||||||
|
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||||
|
</pattern> |
||||||
|
<charset>UTF-8</charset> |
||||||
|
</encoder> |
||||||
|
</appender> |
||||||
|
|
||||||
|
<conversionRule conversionWord="messsage" |
||||||
|
converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/> |
||||||
|
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender"> |
||||||
|
<filter class="ch.qos.logback.classic.filter.ThresholdFilter"> |
||||||
|
<level>INFO</level> |
||||||
|
</filter> |
||||||
|
<filter class="org.apache.dolphinscheduler.server.log.TaskLogFilter"/> |
||||||
|
<Discriminator class="org.apache.dolphinscheduler.server.log.TaskLogDiscriminator"> |
||||||
|
<key>taskAppId</key> |
||||||
|
<logBase>${log.base}</logBase> |
||||||
|
</Discriminator> |
||||||
|
<sift> |
||||||
|
<appender name="FILE-${taskAppId}" class="ch.qos.logback.core.FileAppender"> |
||||||
|
<file>${log.base}/${taskAppId}.log</file> |
||||||
|
<encoder> |
||||||
|
<pattern> |
||||||
|
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n |
||||||
|
</pattern> |
||||||
|
<charset>UTF-8</charset> |
||||||
|
</encoder> |
||||||
|
<append>true</append> |
||||||
|
</appender> |
||||||
|
</sift> |
||||||
|
</appender> |
||||||
|
<!-- master server logback config start --> |
||||||
|
<appender name="MASTERLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> |
||||||
|
<file>${log.base}/dolphinscheduler-master.log</file> |
||||||
|
<!--<filter class="org.apache.dolphinscheduler.server.log.MasterLogFilter"> |
||||||
|
<level>INFO</level> |
||||||
|
</filter>--> |
||||||
|
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> |
||||||
|
<fileNamePattern>${log.base}/dolphinscheduler-master.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> |
||||||
|
<maxHistory>168</maxHistory> |
||||||
|
<maxFileSize>200MB</maxFileSize> |
||||||
|
</rollingPolicy> |
||||||
|
<encoder> |
||||||
|
<pattern> |
||||||
|
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n |
||||||
|
</pattern> |
||||||
|
<charset>UTF-8</charset> |
||||||
|
</encoder> |
||||||
|
</appender> |
||||||
|
<!-- master server logback config end --> |
||||||
|
|
||||||
|
<root level="INFO"> |
||||||
|
<appender-ref ref="STDOUT"/> |
||||||
|
<appender-ref ref="TASKLOGFILE"/> |
||||||
|
<appender-ref ref="MASTERLOGFILE"/> |
||||||
|
</root> |
||||||
|
|
||||||
|
</configuration> |
||||||
Loading…
Reference in new issue