Repository navigation
Expand file tree
/
Copy path.travis.yml
More file actions
723 lines (614 loc) · 33.8 KB
/
Copy path.travis.yml
File metadata and controls
723 lines (614 loc) · 33.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
language: java
dist: xenial
jdk:
- openjdk8
cache:
directories:
- $HOME/.m2
env:
global:
- DOCKER_IP=127.0.0.1 # for integration tests
- MVN="mvn -B"
- > # Various options to make execution of maven goals faster (e.g., mvn install)
MAVEN_SKIP="-Pskip-static-checks -Ddruid.console.skip=true -Dmaven.javadoc.skip=true"
- MAVEN_SKIP_TESTS="-Pskip-tests"
addons:
apt:
packages:
- maven
- python3
# Add various options to make 'mvn install' fast and skip javascript compile (-Ddruid.console.skip=true) since it is not
# needed. Depending on network speeds, "mvn -q install" may take longer than the default 10 minute timeout to print any
# output. To compensate, use travis_wait to extend the timeout.
install: ./check_test_suite.py && travis_terminate 0 || echo 'Running maven install...' && MAVEN_OPTS='-Xmx3000m' travis_wait 15 ${MVN} clean install -q -ff -pl '!distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
# There are 3 stages of tests
# 1. Tests - phase 1
# 2. Tests - phase 2
# 3. cron
#
# The cron type only runs jobs that are marked with stage cron.
# The test type is split into 2 stages. This is done so that more PRs can run their validation
# in parallel. The first phase is meant to include sanity test jobs. The jobs in this phase are
# meant to be fast. The second phase is meant to run all other tests.
# Jobs with known flaky tests should be put in the second phase since the second phase will not
# start if there are any failures in the second stage.
stages:
- name: Tests - phase 1
if: type != cron
- name: Tests - phase 2
if: type != cron
- name: cron
if: type = cron
jobs:
include:
- name: "animal sniffer checks"
stage: Tests - phase 1
script: ${MVN} animal-sniffer:check --fail-at-end
- name: "checkstyle"
script: ${MVN} checkstyle:checkstyle --fail-at-end
- name: "enforcer checks"
script: ${MVN} enforcer:enforce --fail-at-end
- name: "forbidden api checks"
script: ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end
- name: "pmd checks"
script: ${MVN} pmd:check --fail-at-end # TODO: consider adding pmd:cpd-check
- name: "spotbugs checks"
script: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks'
- name: "license checks"
install: skip
before_script: &setup_generate_license
- sudo apt-get update && sudo apt-get install python3 python3-pip python3-setuptools -y
- ./check_test_suite.py && travis_terminate 0 || echo 'Continuing setup'
- pip3 install wheel # install wheel first explicitly
- pip3 install pyyaml
script:
- >
${MVN} apache-rat:check -Prat --fail-at-end
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
-Drat.consoleOutput=true
# Generate dependency reports and checks they are valid. When running on Travis CI, 2 cores are available
# (https://docs.travis-ci.com/user/reference/overview/#virtualisation-environment-vs-operating-system).
- mkdir -p target
- distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2
- distribution/bin/check-licenses.py licenses.yaml target/license-reports
- name: "script checks"
install: skip
# who watches the watchers?
script: ./check_test_suite_test.py
- name: "(openjdk8) strict compilation"
install: skip
# Strict compilation requires more than 2 GB
script: >
./check_test_suite.py && travis_terminate 0 || MAVEN_OPTS='-Xmx3000m' ${MVN} clean -Pstrict compile test-compile --fail-at-end
-pl '!benchmarks' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
- name: "analyze dependencies"
script: |-
MAVEN_OPTS='-Xmx3000m' ${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true || { echo "
The dependency analysis has found a dependency that is either:
1) Used and undeclared: These are available as a transitive dependency but should be explicitly
added to the POM to ensure the dependency version. The XML to add the dependencies to the POM is
shown above.
2) Unused and declared: These are not needed and removing them from the POM will speed up the build
and reduce the artifact size. The dependencies to remove are shown above.
If there are false positive dependency analysis warnings, they can be suppressed:
https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html#usedDependencies
https://maven.apache.org/plugins/maven-dependency-plugin/examples/exclude-dependencies-from-dependency-analysis.html
For more information, refer to:
https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html
" && false; }
- name: "intellij inspections"
script: >
./check_test_suite.py && travis_terminate 0 || docker run --rm
-v $(pwd):/project
-v ~/.m2:/home/inspect/.m2
ccaominh/intellij-inspect:1.0.0
/project/pom.xml
/project/.idea/inspectionProfiles/Druid.xml
--levels ERROR
--scope JavaInspectionsScope
- &package
name: "(openjdk8) packaging check"
install: skip
before_script: *setup_generate_license
script: >
MAVEN_OPTS='-Xmx3000m' ${MVN} clean install -Prat -Pdist -Pbundle-contrib-exts --fail-at-end
-pl '!benchmarks' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -Ddruid.console.skip=false -T1C
- <<: *package
name: "(openjdk11) packaging check"
stage: Tests - phase 2
jdk: openjdk11
- <<: *package
name: "(openjdk15) packaging check"
stage: Tests - phase 2
jdk: openjdk15
- &test_processing_module
name: "(openjdk8) processing module test"
stage: Tests - phase 1
env:
- MAVEN_PROJECTS='processing'
before_script:
- export DRUID_USE_DEFAULT_VALUE_FOR_NULL=true
script:
- unset _JAVA_OPTIONS
# Set MAVEN_OPTS for Surefire launcher. Skip remoteresources to avoid intermittent connection timeouts when
# resolving the SIGAR dependency.
- >
MAVEN_OPTS='-Xmx1100m' ${MVN} test -pl ${MAVEN_PROJECTS}
${MAVEN_SKIP} -Dremoteresources.skip=true -Ddruid.generic.useDefaultValueForNull=${DRUID_USE_DEFAULT_VALUE_FOR_NULL}
- sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
- free -m
- ${MVN} -pl ${MAVEN_PROJECTS} jacoco:report
# Add merge target branch to determine diff (see https://github.com/travis-ci/travis-ci/issues/6069).
# This is not needed for build triggered by tags, since there will be no code diff.
- echo "TRAVIS_BRANCH=${TRAVIS_BRANCH}" # for debugging
- if [[ -z "${TRAVIS_TAG}" ]]; then git remote set-branches --add origin ${TRAVIS_BRANCH} && git fetch; fi
# Determine the modified files that match the maven projects being tested. We use maven project lists that
# either exclude (starts with "!") or include (does not start with "!"), so both cases need to be handled.
# If the build is triggered by a tag, an error will be printed, but `all_files` will be correctly set to empty
# so that the coverage check is skipped.
- all_files="$(git diff --name-only origin/${TRAVIS_BRANCH}...HEAD | grep "\.java$" || [[ $? == 1 ]])"
- for f in ${all_files}; do echo $f; done # for debugging
- >
if [[ "${MAVEN_PROJECTS}" = \!* ]]; then
regex="${MAVEN_PROJECTS:1}";
regex="^${regex//,\!/\\|^}";
project_files="$(echo "${all_files}" | grep -v "${regex}" || [[ $? == 1 ]])";
else
regex="^${MAVEN_PROJECTS//,/\\|^}";
project_files="$(echo "${all_files}" | grep "${regex}" || [[ $? == 1 ]])";
fi
- for f in ${project_files}; do echo $f; done # for debugging
# Check diff code coverage for the maven projects being tested (retry install in case of network error).
# Currently, the function coverage check is not reliable, so it is disabled.
- >
if [ -n "${project_files}" ]; then
travis_retry npm install @connectis/diff-test-coverage@1.5.3
&& git diff origin/${TRAVIS_BRANCH}...HEAD -- ${project_files}
| node_modules/.bin/diff-test-coverage
--coverage "**/target/site/jacoco/jacoco.xml"
--type jacoco
--line-coverage 50
--branch-coverage 50
--function-coverage 0
--log-template "coverage-lines-complete"
--log-template "coverage-files-complete"
--log-template "totals-complete"
--log-template "errors"
--
|| { printf "\n\n****FAILED****\nDiff code coverage check failed.
7438
To view coverage report, run 'mvn clean test jacoco:report' and open 'target/site/jacoco/index.html'\nFor more details on how to run code coverage locally, follow instructions here - https://github.com/apache/druid/blob/master/dev/code-review/code-coverage.md#running-code-coverage-locally\n\n" && false; }
fi
after_success:
# retry in case of network error
- travis_retry curl -o codecov.sh -s https://codecov.io/bash
- travis_retry bash codecov.sh -X gcov
- <<: *test_processing_module
name: "(openjdk11) processing module test"
stage: Tests - phase 2
jdk: openjdk11
- <<: *test_processing_module
name: "(openjdk15) processing module test"
stage: Tests - phase 2
jdk: openjdk15
- &test_processing_module_sqlcompat
<<: *test_processing_module
name: "(openjdk8) processing module test (SQL Compatibility)"
stage: Tests - phase 1
before_script: &setup_sqlcompat
- export DRUID_USE_DEFAULT_VALUE_FOR_NULL=false
- <<: *test_processing_module_sqlcompat
name: "(openjdk11) processing module test (SQL Compatibility)"
stage: Tests - phase 2
jdk: openjdk11
- <<: *test_processing_module_sqlcompat
name: "(openjdk15) processing module test (SQL Compatibility)"
stage: Tests - phase 2
jdk: openjdk15
- &test_indexing_module
<<: *test_processing_module
name: "(openjdk8) indexing modules test"
env:
- MAVEN_PROJECTS='indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service'
- <<: *test_indexing_module
name: "(openjdk11) indexing modules test"
stage: Tests - phase 2
jdk: openjdk11
- <<: *test_indexing_module
name: "(openjdk15) indexing modules test"
stage: Tests - phase 2
jdk: openjdk15
- &test_indexing_module_sqlcompat
<<: *test_indexing_module
name: "(openjdk8) indexing modules test (SQL Compatibility)"
stage: Tests - phase 1
before_script: *setup_sqlcompat
- <<: *test_indexing_module_sqlcompat
name: "(openjdk11) indexing modules test (SQL Compatibility)"
stage: Tests - phase 2
jdk: openjdk11
- <<: *test_indexing_module_sqlcompat
name: "(openjdk15) indexing modules test (SQL Compatibility)"
stage: Tests - phase 2
jdk: openjdk15
- &test_server_module
<<: *test_processing_module
name: "(openjdk8) server module test"
env:
- MAVEN_PROJECTS='server'
- <<: *test_server_module
name: "(openjdk11) server module test"
stage: Tests - phase 2
jdk: openjdk11
- <<: *test_server_module
name: "(openjdk15) server module test"
stage: Tests - phase 2
jdk: openjdk15
- &test_server_module_sqlcompat
<<: *test_server_module
name: "(openjdk8) server module test (SQL Compatibility)"
before_script: *setup_sqlcompat
- <<: *test_server_module_sqlcompat
name: "(openjdk11) server module test (SQL Compatibility)"
stage: Tests - phase 2
jdk: openjdk11
- <<: *test_server_module_sqlcompat
name: "(openjdk15) server module test (SQL Compatibility)"
stage: Tests - phase 2
jdk: openjdk15
- &test_other_modules
<<: *test_processing_module
name: "(openjdk8) other modules test"
env:
- MAVEN_PROJECTS='!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests'
- <<: *test_other_modules
name: "(openjdk11) other modules test"
stage: Tests - phase 2
jdk: openjdk11
- <<: *test_other_modules
name: "(openjdk15) other modules test"
stage: Tests - phase 2
jdk: openjdk15
- &test_other_modules_sqlcompat
<<: *test_other_modules
name: "(openjdk8) other modules test (SQL Compatibility)"
before_script: *setup_sqlcompat
- <<: *test_other_modules_sqlcompat
name: "(openjdk11) other modules test (SQL Compatibility)"
stage: Tests - phase 2
jdk: openjdk11
- <<: *test_other_modules_sqlcompat
name: "(openjdk15) other modules test (SQL Compatibility)"
stage: Tests - phase 2
jdk: openjdk15
- name: "web console"
install: skip
stage: Tests - phase 1
script:
- ./check_test_suite.py && travis_terminate 0 || ${MVN} test -pl 'web-console'
after_success:
- (cd web-console && travis_retry npm run codecov) # retry in case of network error
- name: "web console end-to-end test"
before_install: *setup_generate_license
install: web-console/script/druid build
before_script:
- ./check_test_suite.py && travis_terminate 0 || echo 'Starting nvm install...'
- nvm install 10.24.0
- web-console/script/druid start
script: (cd web-console && npm run test-e2e)
after_script: web-console/script/druid stop
- name: "docs"
install: ./check_test_suite.py && travis_terminate 0 || (cd website && npm install)
script: |-
(cd website && npm run lint && npm run spellcheck) || { echo "
If there are spell check errors:
1) Suppressing False Positives: Edit website/.spelling to add suppressions. Instructions
are at the top of the file and explain how to suppress false positives either globally or
within a particular file.
2) Running Spell Check Locally: cd website && npm install && npm run spellcheck
For more information, refer to: https://www.npmjs.com/package/markdown-spellcheck
" && false; }
# Integration tests Java Compile version is set by the machine environment jdk (set by the jdk key)
# Integration tests Java Runtime version is set by the JVM_RUNTIME env property (set env key to -Djvm.runtime=<JVM_RUNTIME_VERSION>)
# Integration tests will either use MiddleManagers or Indexers
# (Currently integration tests only support running with jvm runtime 8 and 11)
# START - Integration tests for Compile with Java 8 and Run with Java 8
- &integration_batch_index
name: "(Compile=openjdk8, Run=openjdk8) batch index integration test"
stage: Tests - phase 2
jdk: openjdk8
services: &integration_test_services
- docker
env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: &run_integration_test
- ${MVN} verify -pl integration-tests -P integration-tests ${TESTNG_GROUPS} ${JVM_RUNTIME} -Dit.indexer=${USE_INDEXER} ${MAVEN_SKIP}
after_failure: &integration_test_diags
- for v in ~/shared/logs/*.log ; do
echo $v logtail ======================== ; tail -100 $v ;
done
- for v in broker middlemanager overlord router coordinator historical ; do
echo $v dmesg ======================== ;
docker exec -it druid-$v sh -c 'dmesg | tail -3' ;
done
- <<: *integration_batch_index
name: "(Compile=openjdk8, Run=openjdk8) batch index integration test with Indexer"
env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer'
- &integration_input_format
name: "(Compile=openjdk8, Run=openjdk8) input format integration test"
stage: Tests - phase 2
jdk: openjdk8
services: *integration_test_services
env: TESTNG_GROUPS='-Dgroups=input-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: *run_integration_test
after_failure: *integration_test_diags
- <<: *integration_input_format
name: "(Compile=openjdk8, Run=openjdk8) input format integration test with Indexer"
env: TESTNG_GROUPS='-Dgroups=input-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer'
- &integration_input_source
name: "(Compile=openjdk8, Run=openjdk8) input source integration test"
stage: Tests - phase 2
jdk: openjdk8
services: *integration_test_services
env: TESTNG_GROUPS='-Dgroups=input-source' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: *run_integration_test
after_failure: *integration_test_diags
- <<: *integration_input_source
name: "(Compile=openjdk8, Run=openjdk8) input source integration test with Indexer"
env: TESTNG_GROUPS='-Dgroups=input-source' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer'
- &integration_perfect_rollup_parallel_batch_index
name: "(Compile=openjdk8, Run=openjdk8) perfect rollup parallel batch index integration test"
jdk: openjdk8
stage: Tests - phase 2
services: *integration_test_services
env: TESTNG_GROUPS='-Dgroups=perfect-rollup-parallel-batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: *run_integration_test
after_failure: *integration_test_diags
- <<: *integration_perfect_rollup_parallel_batch_index
name: "(Compile=openjdk8, Run=openjdk8) perfect rollup parallel batch index integration test with Indexer"
env: TESTNG_GROUPS='-Dgroups=perfect-rollup-parallel-batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer'
- &integration_kafka_index
name: "(Compile=openjdk8, Run=openjdk8) kafka index integration test"
stage: Tests - phase 2
jdk: openjdk8
services: *integration_test_services
env: TESTNG_GROUPS='-Dg
8BD0
roups=kafka-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: *run_integration_test
after_failure: *integration_test_diags
- <<: *integration_kafka_index
name: "(Compile=openjdk8, Run=openjdk8) kafka index, transactional kafka index integration test with Indexer"
env: TESTNG_GROUPS='-Dgroups=kafka-index,kafka-transactional-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer'
- &integration_kafka_index_slow
name: "(Compile=openjdk8, Run=openjdk8) kafka index integration test slow"
stage: Tests - phase 2
jdk: openjdk8
services: *integration_test_services
env: TESTNG_GROUPS='-Dgroups=kafka-index-slow' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: *run_integration_test
after_failure: *integration_test_diags
- <<: *integration_kafka_index_slow
name: "(Compile=openjdk8, Run=openjdk8) kafka index integration test slow with Indexer"
env: TESTNG_GROUPS='-Dgroups=kafka-index-slow' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer'
- &integration_kafka_transactional_index
name: "(Compile=openjdk8, Run=openjdk8) transactional kafka index integration test"
stage: Tests - phase 2
jdk: openjdk8
services: *integration_test_services
env: TESTNG_GROUPS='-Dgroups=kafka-transactional-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: *run_integration_test
after_failure: *integration_test_diags
- &integration_kafka_transactional_index_slow
name: "(Compile=openjdk8, Run=openjdk8) transactional kafka index integration test slow"
stage: Tests - phase 2
jdk: openjdk8
services: *integration_test_services
env: TESTNG_GROUPS='-Dgroups=kafka-transactional-index-slow' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: *run_integration_test
after_failure: *integration_test_diags
- <<: *integration_kafka_transactional_index_slow
name: "(Compile=openjdk8, Run=openjdk8) transactional kafka index integration test slow with Indexer"
env: TESTNG_GROUPS='-Dgroups=kafka-transactional-index-slow' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer'
- &integration_kafka_format_tests
name: "(Compile=openjdk8, Run=openjdk8) Kafka index integration test with various formats"
stage: Tests - phase 2
jdk: openjdk8
services: *integration_test_services
env: TESTNG_GROUPS='-Dgroups=kafka-data-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: *run_integration_test
after_failure: *integration_test_diags
- <<: *integration_kafka_format_tests
name: "(Compile=openjdk8, Run=openjdk8) Kafka index integration test with various formats with Indexer"
env: TESTNG_GROUPS='-Dgroups=kafka-data-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer'
- &integration_query
name: "(Compile=openjdk8, Run=openjdk8) query integration test"
stage: Tests - phase 2
jdk: openjdk8
services: *integration_test_services
env: TESTNG_GROUPS='-Dgroups=query' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: *run_integration_test
after_failure: *integration_test_diags
- &integration_query_retry
name: "(Compile=openjdk8, Run=openjdk8) query retry integration test for missing segments"
stage: Tests - phase 2
jdk: openjdk8
services: *integration_test_services
env: TESTNG_GROUPS='-Dgroups=query-retry' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: *run_integration_test
after_failure: *integration_test_diags
- &integration_query_error
name: "(Compile=openjdk8, Run=openjdk8) query error integration test"
stage: Tests - phase 2
jdk: openjdk8
services: *integration_test_services
env: TESTNG_GROUPS='-Dgroups=query-error' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: *run_integration_test
after_failure: *integration_test_diags
- &integration_security
name: "(Compile=openjdk8, Run=openjdk8) security integration test"
stage: Tests - phase 2
jdk: openjdk8
services: *integration_test_services
env: TESTNG_GROUPS='-Dgroups=security' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: *run_integration_test
after_failure: *integration_test_diags
- &integration_ldap_security
name: "(Compile=openjdk8, Run=openjdk8) ldap security integration test"
stage: Tests - phase 2
jdk: openjdk8
services: *integration_test_services
env: TESTNG_GROUPS='-Dgroups=ldap-security' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: *run_integration_test
after_failure: *integration_test_diags
- &integration_realtime_index
name: "(Compile=openjdk8, Run=openjdk8) realtime index integration test"
stage: Tests - phase 2
jdk: openjdk8
services: *integration_test_services
env: TESTNG_GROUPS='-Dgroups=realtime-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: *run_integration_test
after_failure: *integration_test_diags
- &integration_append_ingestion
name: "(Compile=openjdk8, Run=openjdk8) append ingestion integration test"
stage: Tests - phase 2
jdk: openjdk8
services: *integration_test_services
env: TESTNG_GROUPS='-Dgroups=append-ingestion' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: *run_integration_test
after_failure: *integration_test_diags
- <<: *integration_append_ingestion
name: "(Compile=openjdk8, Run=openjdk8) append ingestion integration test with Indexer"
env: TESTNG_GROUPS='-Dgroups=append-ingestion' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer'
- &integration_compaction_tests
name: "(Compile=openjdk8, Run=openjdk8) compaction integration test"
stage: Tests - phase 2
jdk: openjdk8
services: *integration_test_services
env: TESTNG_GROUPS='-Dgroups=compaction' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: *run_integration_test
after_failure: *integration_test_diags
- <<: *integration_compaction_tests
name: "(Compile=openjdk8, Run=openjdk8) compaction integration test with Indexer"
env: TESTNG_GROUPS='-Dgroups=compaction' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer'
- &integration_tests
name: "(Compile=openjdk8, Run=openjdk8) other integration tests"
stage: Tests - phase 2
jdk: openjdk8
services: *integration_test_services
env: TESTNG_GROUPS='-DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: *run_integration_test
after_failure: *integration_test_diags
- <<: *integration_tests
name: "(Compile=openjdk8, Run=openjdk8) other integration tests with Indexer"
env: TESTNG_GROUPS='-DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer'
- <<: *integration_tests
name: "(Compile=openjdk8, Run=openjdk8) leadership and high availability integration tests"
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
# END - Integration tests for Compile with Java 8 and Run with Java 8
# START - Integration tests for Compile with Java 8 and Run with Java 11
- <<: *integration_batch_index
name: "(Compile=openjdk8, Run=openjdk11) batch index integration test"
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
- <<: *integration_input_format
name: "(Compile=openjdk8, Run=openjdk11) input format integration test"
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=input-format' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
- <<: *integration_input_source
name: "(Compile=openjdk8, Run=openjdk11) input source integration test"
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=input-source' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
- <<: *integration_perfect_rollup_parallel_batch_index
name: "(Compile=openjdk8, Run=openjdk11) perfect rollup parallel batch index integration test"
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=perfect-rollup-parallel-batch-index' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
- <<: *integration_query
name: "(Compile=openjdk8, Run=openjdk11) query integration test"
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=query' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
- <<: *integration_query_retry
name: "(Compile=openjdk8, Run=openjdk11) query retry integration test for missing segments"
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=query-retry' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
- <<: *integration_query_error
name: "(Compile=openjdk8, Run=openjdk11) query error integration test for missing segments"
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=query-error' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
- <<: *integration_security
name: "(Compile=openjdk8, Run=openjdk11) security integration test"
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=security' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
- <<: *integration_ldap_security
name: "(Compile=openjdk8, Run=openjdk11) ldap security integration test"
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=ldap-security' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
- <<: *integration_realtime_index
name: "(Compile=openjdk8, Run=openjdk11) realtime index integration test"
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=realtime-index' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
- <<: *integration_append_ingestion
name: "(Compile=openjdk8, Run=openjdk11) append ingestion integration test"
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=append-ingestion' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
- <<: *integration_compaction_tests
name: "(Compile=openjdk8, Run=openjdk11) compaction integration test"
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=compaction' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
- <<: *integration_tests
name: "(Compile=openjdk8, Run=openjdk11) other integration test"
jdk: openjdk8
env: TESTNG_GROUPS='-DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
- <<: *integration_tests
name: "(Compile=openjdk8, Run=openjdk11) leadership and high availability integration tests"
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
# END - Integration tests for Compile with Java 8 and Run with Java 11
- &integration_batch_index_k8s
name: "(Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) ITNestedQueryPushDownTest integration test"
stage: Tests - phase 2
jdk: openjdk8
services: &integration_test_services_k8s
- docker
env: CONFIG_FILE='k8s_run_config_file.json' IT_TEST='-Dit.test=ITNestedQueryPushDownTest' POD_NAME=int-test POD_NAMESPACE=default BUILD_DRUID_CLSUTER=true
script: &run_integration_test_k8s
- ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP} -Dpod.name=${POD_NAME} -Dpod.namespace=${POD_NAMESPACE} -Dbuild.druid.cluster=${BUILD_DRUID_CLSUTER}
after_failure: &integration_test_diags_k8s
- for v in broker middlemanager router coordinator historical ; do
echo "------------------------druid-tiny-cluster-"$v"s-0-------------------------";
sudo /usr/local/bin/kubectl logs --tail 1000 druid-tiny-cluster-"$v"s-0;
done
- name: "security vulnerabilities"
stage: cron
install: skip
script: |-
${MVN} dependency-check:aggregate -pl '!integration-tests' || { echo "
The OWASP dependency check has found security vulnerabilities. Please use a newer version
of the dependency that does not have vulnerabilities. To see a report run
`mvn dependency-check:check`
If the analysis has false positives,
they can be suppressed by adding entries to owasp-dependency-check-suppressions.xml (for more
information, see https://jeremylong.github.io/DependencyCheck/general/suppression.html).
" && false; }
# Travis CI only supports per build (and not per-job notifications): https://github.com/travis-ci/travis-ci/issues/9888
notifications:
email:
if: type = cron
recipients:
# This is the string "dev@druid.apache.org" encrypted against the apache/druid repo so that forks are unable to
# use this notification:
# https://github.com/travis-ci/travis-ci/issues/1094#issuecomment-215019909
# https://github.com/travis-ci/travis-ci/issues/2711
- secure: "MupjX/0jLwh3XzHPl74BTk2/Kp5r+8TrEewfRhpQdWKFMBXLKNqu0k2VXf5C/NIg3uvPianq3REk+qeTHI8dL2ShjiWS/eIRkJOHLfObdNNBuos5fo4TxAuBQcXyT4VjAq5jnAkH84Pxf2Nl0rkisWoIhvwSX7+kNrjW1qdu7K0="
on_success: change
on_failure: change