digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (6)\n \nduration: 0 ms";
2 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 0 ms<br>peak memory: 64.2 MiB<br>number of output rows: 1<br>number of sort fallback tasks: 0<br>avg hash probes per key: 1"];
}
3 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 251.0 B<br>number of coalesced partitions: 1"];
4 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 1<br>local merged chunks fetched: 0<br>shuffle write time: 0 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 1<br>local bytes read: 244.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 240.0 B<br>local merged bytes read: 0.0 B<br>number of partitions: 37<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 244.0 B"];
subgraph cluster5 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: 13 ms";
6 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 4 ms<br>peak memory: 64.2 MiB<br>number of output rows: 1<br>number of sort fallback tasks: 0<br>avg hash probes per key: 1"];
7 [labelType="html" label="<br><b>Project</b><br><br>"];
8 [labelType="html" label="<b>BroadcastHashJoin</b><br><br>number of output rows: 1"];
15 [labelType="html" label="<br><b>Project</b><br><br>"];
16 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 1"];
17 [labelType="html" label="<b>Generate</b><br><br>number of output rows: 1"];
18 [labelType="html" label="<br><b>Project</b><br><br>"];
19 [labelType="html" label="<b>BroadcastHashJoin</b><br><br>number of output rows: 1"];
26 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 3 ms<br>peak memory: 256.0 KiB<br>number of output rows: 1<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0"];
}
9 [labelType="html" label="<b>BroadcastExchange</b><br><br>time to broadcast: 7 ms<br>time to build: 6 ms<br>time to collect: 363 ms<br>number of output rows: 10<br>data size: 64.0 MiB"];
10 [labelType="html" label="<br><b>Project</b><br><br>"];
subgraph cluster11 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n1.5 s (157 ms, 216 ms, 256 ms (stage 134.0: task 1084))";
12 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 10"];
13 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 10<br>number of input batches: 7"];
}
14 [labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 7<br>scan time total (min, med, max (stageId: taskId))<br>1.5 s (154 ms, 214 ms, 252 ms (stage 134.0: task 1084))<br>metadata time: 0 ms<br>size of files read: 49.6 KiB<br>number of output rows: 10"];
20 [labelType="html" label="<b>BroadcastExchange</b><br><br>time to broadcast: 0 ms<br>time to build: 6 ms<br>time to collect: 22 ms<br>number of output rows: 10<br>data size: 64.0 MiB"];
21 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1"];
22 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 10<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>6 ms (0 ms, 0 ms, 0 ms (stage 132.0: task 1076))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 10<br>local bytes read: 520.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 788.0 B<br>merged fetch fallback count: 0<br>local blocks read: 4<br>remote merged chunks fetched: 0<br>remote blocks read: 6<br>data size total (min, med, max (stageId: taskId))<br>920.0 B (80.0 B, 96.0 B, 104.0 B (stage 132.0: task 1071))<br>local merged bytes read: 0.0 B<br>number of partitions: 37<br>remote reqs duration: 2 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>1308.0 B (123.0 B, 132.0 B, 136.0 B (stage 132.0: task 1075))"];
subgraph cluster23 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n1.2 s (105 ms, 117 ms, 150 ms (stage 132.0: task 1075))";
24 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 10"];
25 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 10"];
}
27 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 2.4 KiB<br>number of coalesced partitions: 1"];
28 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 25<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>14 ms (0 ms, 0 ms, 0 ms (stage 137.0: task 1111))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 25<br>local bytes read: 828.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 1472.0 B<br>merged fetch fallback count: 0<br>local blocks read: 9<br>remote merged chunks fetched: 0<br>remote blocks read: 16<br>data size total (min, med, max (stageId: taskId))<br>800.0 B (32.0 B, 32.0 B, 32.0 B (stage 137.0: task 1096))<br>local merged bytes read: 0.0 B<br>number of partitions: 37<br>remote reqs duration: 3 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>2.2 KiB (92.0 B, 92.0 B, 92.0 B (stage 137.0: task 1096))"];
subgraph cluster29 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n2.3 s (79 ms, 91 ms, 113 ms (stage 137.0: task 1094))";
30 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build total (min, med, max (stageId: taskId))<br>2.2 s (76 ms, 85 ms, 105 ms (stage 137.0: task 1094))<br>peak memory total (min, med, max (stageId: taskId))<br>6.3 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 137.0: task 1096))<br>number of output rows: 25<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0"];
31 [labelType="html" label="<br><b>Project</b><br><br>"];
32 [labelType="html" label="<b>BroadcastHashJoin</b><br><br>number of output rows: 1,174"];
33 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 1,196<br>number of input batches: 25"];
}
34 [labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 25<br>scan time total (min, med, max (stageId: taskId))<br>2.2 s (76 ms, 85 ms, 105 ms (stage 137.0: task 1094))<br>metadata time: 0 ms<br>size of files read: 83.2 KiB<br>number of output rows: 1,196"];
35 [labelType="html" label="<b>BroadcastExchange</b><br><br>time to broadcast: 1 ms<br>time to build: 7 ms<br>time to collect: 193 ms<br>number of output rows: 1,174<br>data size: 64.1 MiB"];
subgraph cluster36 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: 168 ms";
37 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 1,174"];
38 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 1,196<br>number of input batches: 1"];
}
39 [labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 1<br>scan time: 167 ms<br>metadata time: 0 ms<br>size of files read: 8.1 KiB<br>number of output rows: 1,196"];
2->0;
3->2;
4->3;
6->4;
7->6;
8->7;
9->8;
10->9;
12->10;
13->12;
14->13;
15->8;
16->15;
17->16;
18->17;
19->18;
20->19;
21->20;
22->21;
24->22;
25->24;
26->19;
27->26;
28->27;
30->28;
31->30;
32->31;
33->32;
34->33;
35->32;
37->35;
38->37;
39->38;
}
40
AdaptiveSparkPlan isFinalPlan=true
HashAggregate(keys=[name#3594, considerVariance#3603, customerOrderItemTypeCodes#3639, customerOrderTypes#3627, distributionChains#3614, forecastCode#3597, demandStream#3596], functions=[])
WholeStageCodegen (6)
AQEShuffleRead coalesced
Exchange hashpartitioning(name#3594, considerVariance#3603, customerOrderItemTypeCodes#3639, customerOrderTypes#3627, distributionChains#3614, forecastCode#3597, demandStream#3596, 37), ENSURE_REQUIREMENTS, [plan_id=5502]
HashAggregate(keys=[name#3594, considerVariance#3603, customerOrderItemTypeCodes#3639, customerOrderTypes#3627, distributionChains#3614, forecastCode#3597, demandStream#3596], functions=[])
Project [name#3594, demandStream#3596, forecastCode#3597, distributionChains#3614, customerOrderTypes#3627, customerOrderItemTypeCodes#3639, considerVariance#3603]
BroadcastHashJoin [name#3594], [name#3588], Inner, BuildLeft, false
Project [explodedDemandChannels#3582.name AS name#3588]
Filter isnotnull(explodedDemandChannels#3582.name)
Generate explode(demandChannels#3570), false, [explodedDemandChannels#3582]
Project [demandChannels#3570]
BroadcastHashJoin [demandProfile#3569], [demandProfile#3418], Inner, BuildLeft, false
HashAggregate(keys=[demandProfile#3418], functions=[])
WholeStageCodegen (5)
BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5092]
Project [name#3594, demandStream#3596, forecastCode#3597, transform(distributionChains#3599, lambdafunction(named_struct(internalUUID, lambda chain#3615.internalUUID, salesOrganizationUUID, lambda chain#3615.salesOrganization.internalRefUUID, distributionChannelUUID, lambda chain#3615.distributionChannel.internalRefUUID), lambda chain#3615, false)) AS distributionChains#3614, transform(customerOrderTypes#3600, lambdafunction(named_struct(internalUUID, lambda documentType#3628.internalRefUUID), lambda documentType#3628, false)) AS customerOrderTypes#3627, transform(customerOrderItemTypeCodes#3601, lambdafunction(named_struct(customerOrderItemTypeCodeUUID, lambda itemType#3640.internalRefUUID), lambda itemType#3640, false)) AS customerOrderItemTypeCodes#3639, considerVariance#3603]
Filter isnotnull(name#3594)
ColumnarToRow
WholeStageCodegen (1)
FileScan parquet [name#3594,demandStream#3596,forecastCode#3597,distributionChains#3599,customerOrderTypes#3600,customerOrderItemTypeCodes#3601,considerVariance#3603] Batched: true, DataFilters: [isnotnull(name#3594)], Format: Parquet, Location: PreparedDeltaFileIndex(1 paths)[hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20...., PartitionFilters: [], PushedFilters: [IsNotNull(name)], ReadSchema: struct<name:string,demandStream:int,forecastCode:string,distributionChains:array<struct<internalU...
BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false]),false), [plan_id=5232]
AQEShuffleRead local
Exchange hashpartitioning(demandProfile#3569, 37), ENSURE_REQUIREMENTS, [plan_id=5103]
Filter (isnotnull(demandProfile#3569) AND ((size(demandChannels#3570, true) > 0) AND isnotnull(demandChannels#3570)))
Scan ExistingRDD[demandProfile#3569,demandChannels#3570]
WholeStageCodegen (2)
AQEShuffleRead coalesced
Exchange hashpartitioning(demandProfile#3418, 37), ENSURE_REQUIREMENTS, [plan_id=5334]
HashAggregate(keys=[demandProfile#3418], functions=[])
Project [demandProfile#3418]
BroadcastHashJoin [coalesce(plant#3408, ), isnull(plant#3408), coalesce(product#3407, ), isnull(product#3407)], [coalesce(plant#3416, ), isnull(plant#3416), coalesce(product#3415, ), isnull(product#3415)], Inner, BuildRight, false
ColumnarToRow
WholeStageCodegen (4)
FileScan parquet [product#3407,plant#3408] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20.hanac..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<product:string,plant:string>
BroadcastExchange HashedRelationBroadcastMode(List(coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[0, string, true], ), isnull(input[0, string, true])),false), [plan_id=5116]
Filter isnotnull(demandProfile#3418)
ColumnarToRow
WholeStageCodegen (3)
FileScan parquet [product#3415,plant#3416,demandprofile#3418] Batched: true, DataFilters: [isnotnull(demandprofile#3418)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20.hanac..., PartitionFilters: [], PushedFilters: [IsNotNull(demandprofile)], ReadSchema: struct<product:string,plant:string,demandprofile:string>
== Physical Plan ==
AdaptiveSparkPlan (64)
+- == Final Plan ==
* HashAggregate (39)
+- AQEShuffleRead (38)
+- ShuffleQueryStage (37), Statistics(sizeInBytes=240.0 B, rowCount=1)
+- Exchange (36)
+- * HashAggregate (35)
+- * Project (34)
+- * BroadcastHashJoin Inner BuildLeft (33)
:- BroadcastQueryStage (6), Statistics(sizeInBytes=64.0 MiB, rowCount=10)
: +- BroadcastExchange (5)
: +- Project (4)
: +- * Filter (3)
: +- * ColumnarToRow (2)
: +- Scan parquet (1)
+- * Project (32)
+- * Filter (31)
+- * Generate (30)
+- * Project (29)
+- * BroadcastHashJoin Inner BuildLeft (28)
:- BroadcastQueryStage (13), Statistics(sizeInBytes=64.0 MiB, rowCount=10)
: +- BroadcastExchange (12)
: +- AQEShuffleRead (11)
: +- ShuffleQueryStage (10), Statistics(sizeInBytes=920.0 B, rowCount=10)
: +- Exchange (9)
: +- * Filter (8)
: +- * Scan ExistingRDD (7)
+- * HashAggregate (27)
+- AQEShuffleRead (26)
+- ShuffleQueryStage (25), Statistics(sizeInBytes=800.0 B, rowCount=25)
+- Exchange (24)
+- * HashAggregate (23)
+- * Project (22)
+- * BroadcastHashJoin Inner BuildRight (21)
:- * ColumnarToRow (15)
: +- Scan parquet (14)
+- BroadcastQueryStage (20), Statistics(sizeInBytes=64.1 MiB, rowCount=1.17E+3)
+- BroadcastExchange (19)
+- * Filter (18)
+- * ColumnarToRow (17)
+- Scan parquet (16)
+- == Initial Plan ==
HashAggregate (63)
+- Exchange (62)
+- HashAggregate (61)
+- Project (60)
+- BroadcastHashJoin Inner BuildLeft (59)
:- BroadcastExchange (42)
: +- Project (41)
: +- Filter (40)
: +- Scan parquet (1)
+- Project (58)
+- Filter (57)
+- Generate (56)
+- Project (55)
+- SortMergeJoin Inner (54)
:- Sort (44)
: +- Exchange (43)
: +- Filter (8)
: +- Scan ExistingRDD (7)
+- Sort (53)
+- HashAggregate (52)
+- Exchange (51)
+- HashAggregate (50)
+- Project (49)
+- BroadcastHashJoin Inner BuildRight (48)
:- Scan parquet (45)
+- BroadcastExchange (47)
+- Filter (46)
+- Scan parquet (16)
(1) Scan parquet
Output [7]: [name#3594, demandStream#3596, forecastCode#3597, distributionChains#3599, customerOrderTypes#3600, customerOrderItemTypeCodes#3601, considerVariance#3603]
Batched: true
Location: PreparedDeltaFileIndex [hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-dl-stream-service/prp/demand-channel]
PushedFilters: [IsNotNull(name)]
ReadSchema: struct<name:string,demandStream:int,forecastCode:string,distributionChains:array<struct<internalUUID:string,salesOrganization:struct<internalRefUUID:string,id:string>,distributionChannel:struct<internalRefUUID:string,id:string>>>,customerOrderTypes:array<struct<id:string,internalRefUUID:string>>,customerOrderItemTypeCodes:array<struct<id:string,internalRefUUID:string>>,considerVariance:boolean>
(2) ColumnarToRow [codegen id : 1]
Input [7]: [name#3594, demandStream#3596, forecastCode#3597, distributionChains#3599, customerOrderTypes#3600, customerOrderItemTypeCodes#3601, considerVariance#3603]
(3) Filter [codegen id : 1]
Input [7]: [name#3594, demandStream#3596, forecastCode#3597, distributionChains#3599, customerOrderTypes#3600, customerOrderItemTypeCodes#3601, considerVariance#3603]
Condition : isnotnull(name#3594)
(4) Project
Output [7]: [name#3594, demandStream#3596, forecastCode#3597, transform(distributionChains#3599, lambdafunction(named_struct(internalUUID, lambda chain#3615.internalUUID, salesOrganizationUUID, lambda chain#3615.salesOrganization.internalRefUUID, distributionChannelUUID, lambda chain#3615.distributionChannel.internalRefUUID), lambda chain#3615, false)) AS distributionChains#3614, transform(customerOrderTypes#3600, lambdafunction(named_struct(internalUUID, lambda documentType#3628.internalRefUUID), lambda documentType#3628, false)) AS customerOrderTypes#3627, transform(customerOrderItemTypeCodes#3601, lambdafunction(named_struct(customerOrderItemTypeCodeUUID, lambda itemType#3640.internalRefUUID), lambda itemType#3640, false)) AS customerOrderItemTypeCodes#3639, considerVariance#3603]
Input [7]: [name#3594, demandStream#3596, forecastCode#3597, distributionChains#3599, customerOrderTypes#3600, customerOrderItemTypeCodes#3601, considerVariance#3603]
(5) BroadcastExchange
Input [7]: [name#3594, demandStream#3596, forecastCode#3597, distributionChains#3614, customerOrderTypes#3627, customerOrderItemTypeCodes#3639, considerVariance#3603]
Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5092]
(6) BroadcastQueryStage
Output [7]: [name#3594, demandStream#3596, forecastCode#3597, distributionChains#3614, customerOrderTypes#3627, customerOrderItemTypeCodes#3639, considerVariance#3603]
Arguments: 0
(7) Scan ExistingRDD [codegen id : 2]
Output [2]: [demandProfile#3569, demandChannels#3570]
Arguments: [demandProfile#3569, demandChannels#3570], MapPartitionsRDD[334] at createDataFrame at DemandProfileDataReaderImpl.java:60, ExistingRDD, UnknownPartitioning(0)
(8) Filter [codegen id : 2]
Input [2]: [demandProfile#3569, demandChannels#3570]
Condition : (isnotnull(demandProfile#3569) AND ((size(demandChannels#3570, true) > 0) AND isnotnull(demandChannels#3570)))
(9) Exchange
Input [2]: [demandProfile#3569, demandChannels#3570]
Arguments: hashpartitioning(demandProfile#3569, 37), ENSURE_REQUIREMENTS, [plan_id=5103]
(10) ShuffleQueryStage
Output [2]: [demandProfile#3569, demandChannels#3570]
Arguments: 1
(11) AQEShuffleRead
Input [2]: [demandProfile#3569, demandChannels#3570]
Arguments: local
(12) BroadcastExchange
Input [2]: [demandProfile#3569, demandChannels#3570]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false]),false), [plan_id=5232]
(13) BroadcastQueryStage
Output [2]: [demandProfile#3569, demandChannels#3570]
Arguments: 3
(14) Scan parquet
Output [2]: [product#3407, plant#3408]
Batched: true
Location: InMemoryFileIndex [hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-order-qty-opt-service/out/product-plant-list/10000000091/shardId=0_1_10000000091]
ReadSchema: struct<product:string,plant:string>
(15) ColumnarToRow [codegen id : 4]
Input [2]: [product#3407, plant#3408]
(16) Scan parquet
Output [3]: [product#3415, plant#3416, demandprofile#3418]
Batched: true
Location: InMemoryFileIndex [hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-param-determination-service/out/10000000091/0_1_10000000091/demandprofile]
PushedFilters: [IsNotNull(demandprofile)]
ReadSchema: struct<product:string,plant:string,demandprofile:string>
(17) ColumnarToRow [codegen id : 3]
Input [3]: [product#3415, plant#3416, demandprofile#3418]
(18) Filter [codegen id : 3]
Input [3]: [product#3415, plant#3416, demandprofile#3418]
Condition : isnotnull(demandProfile#3418)
(19) BroadcastExchange
Input [3]: [product#3415, plant#3416, demandprofile#3418]
Arguments: HashedRelationBroadcastMode(List(coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[0, string, true], ), isnull(input[0, string, true])),false), [plan_id=5116]
(20) BroadcastQueryStage
Output [3]: [product#3415, plant#3416, demandprofile#3418]
Arguments: 2
(21) BroadcastHashJoin [codegen id : 4]
Left keys [4]: [coalesce(plant#3408, ), isnull(plant#3408), coalesce(product#3407, ), isnull(product#3407)]
Right keys [4]: [coalesce(plant#3416, ), isnull(plant#3416), coalesce(product#3415, ), isnull(product#3415)]
Join type: Inner
Join condition: None
(22) Project [codegen id : 4]
Output [1]: [demandProfile#3418]
Input [5]: [product#3407, plant#3408, product#3415, plant#3416, demandprofile#3418]
(23) HashAggregate [codegen id : 4]
Input [1]: [demandProfile#3418]
Keys [1]: [demandProfile#3418]
Functions: []
Aggregate Attributes: []
Results [1]: [demandProfile#3418]
(24) Exchange
Input [1]: [demandProfile#3418]
Arguments: hashpartitioning(demandProfile#3418, 37), ENSURE_REQUIREMENTS, [plan_id=5334]
(25) ShuffleQueryStage
Output [1]: [demandProfile#3418]
Arguments: 4
(26) AQEShuffleRead
Input [1]: [demandProfile#3418]
Arguments: coalesced
(27) HashAggregate
Input [1]: [demandProfile#3418]
Keys [1]: [demandProfile#3418]
Functions: []
Aggregate Attributes: []
Results [1]: [demandProfile#3418]
(28) BroadcastHashJoin
Left keys [1]: [demandProfile#3569]
Right keys [1]: [demandProfile#3418]
Join type: Inner
Join condition: None
(29) Project
Output [1]: [demandChannels#3570]
Input [3]: [demandProfile#3569, demandChannels#3570, demandProfile#3418]
(30) Generate
Input [1]: [demandChannels#3570]
Arguments: explode(demandChannels#3570), false, [explodedDemandChannels#3582]
(31) Filter
Input [1]: [explodedDemandChannels#3582]
Condition : isnotnull(explodedDemandChannels#3582.name)
(32) Project
Output [1]: [explodedDemandChannels#3582.name AS name#3588]
Input [1]: [explodedDemandChannels#3582]
(33) BroadcastHashJoin [codegen id : 5]
Left keys [1]: [name#3594]
Right keys [1]: [name#3588]
Join type: Inner
Join condition: None
(34) Project [codegen id : 5]
Output [7]: [name#3594, demandStream#3596, forecastCode#3597, distributionChains#3614, customerOrderTypes#3627, customerOrderItemTypeCodes#3639, considerVariance#3603]
Input [8]: [name#3594, demandStream#3596, forecastCode#3597, distributionChains#3614, customerOrderTypes#3627, customerOrderItemTypeCodes#3639, considerVariance#3603, name#3588]
(35) HashAggregate [codegen id : 5]
Input [7]: [name#3594, demandStream#3596, forecastCode#3597, distributionChains#3614, customerOrderTypes#3627, customerOrderItemTypeCodes#3639, considerVariance#3603]
Keys [7]: [name#3594, considerVariance#3603, customerOrderItemTypeCodes#3639, customerOrderTypes#3627, distributionChains#3614, forecastCode#3597, demandStream#3596]
Functions: []
Aggregate Attributes: []
Results [7]: [name#3594, considerVariance#3603, customerOrderItemTypeCodes#3639, customerOrderTypes#3627, distributionChains#3614, forecastCode#3597, demandStream#3596]
(36) Exchange
Input [7]: [name#3594, considerVariance#3603, customerOrderItemTypeCodes#3639, customerOrderTypes#3627, distributionChains#3614, forecastCode#3597, demandStream#3596]
Arguments: hashpartitioning(name#3594, considerVariance#3603, customerOrderItemTypeCodes#3639, customerOrderTypes#3627, distributionChains#3614, forecastCode#3597, demandStream#3596, 37), ENSURE_REQUIREMENTS, [plan_id=5502]
(37) ShuffleQueryStage
Output [7]: [name#3594, considerVariance#3603, customerOrderItemTypeCodes#3639, customerOrderTypes#3627, distributionChains#3614, forecastCode#3597, demandStream#3596]
Arguments: 5
(38) AQEShuffleRead
Input [7]: [name#3594, considerVariance#3603, customerOrderItemTypeCodes#3639, customerOrderTypes#3627, distributionChains#3614, forecastCode#3597, demandStream#3596]
Arguments: coalesced
(39) HashAggregate [codegen id : 6]
Input [7]: [name#3594, considerVariance#3603, customerOrderItemTypeCodes#3639, customerOrderTypes#3627, distributionChains#3614, forecastCode#3597, demandStream#3596]
Keys [7]: [name#3594, considerVariance#3603, customerOrderItemTypeCodes#3639, customerOrderTypes#3627, distributionChains#3614, forecastCode#3597, demandStream#3596]
Functions: []
Aggregate Attributes: []
Results: []
(40) Filter
Input [7]: [name#3594, demandStream#3596, forecastCode#3597, distributionChains#3599, customerOrderTypes#3600, customerOrderItemTypeCodes#3601, considerVariance#3603]
Condition : isnotnull(name#3594)
(41) Project
Output [7]: [name#3594, demandStream#3596, forecastCode#3597, transform(distributionChains#3599, lambdafunction(named_struct(internalUUID, lambda chain#3615.internalUUID, salesOrganizationUUID, lambda chain#3615.salesOrganization.internalRefUUID, distributionChannelUUID, lambda chain#3615.distributionChannel.internalRefUUID), lambda chain#3615, false)) AS distributionChains#3614, transform(customerOrderTypes#3600, lambdafunction(named_struct(internalUUID, lambda documentType#3628.internalRefUUID), lambda documentType#3628, false)) AS customerOrderTypes#3627, transform(customerOrderItemTypeCodes#3601, lambdafunction(named_struct(customerOrderItemTypeCodeUUID, lambda itemType#3640.internalRefUUID), lambda itemType#3640, false)) AS customerOrderItemTypeCodes#3639, considerVariance#3603]
Input [7]: [name#3594, demandStream#3596, forecastCode#3597, distributionChains#3599, customerOrderTypes#3600, customerOrderItemTypeCodes#3601, considerVariance#3603]
(42) BroadcastExchange
Input [7]: [name#3594, demandStream#3596, forecastCode#3597, distributionChains#3614, customerOrderTypes#3627, customerOrderItemTypeCodes#3639, considerVariance#3603]
Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5028]
(43) Exchange
Input [2]: [demandProfile#3569, demandChannels#3570]
Arguments: hashpartitioning(demandProfile#3569, 37), ENSURE_REQUIREMENTS, [plan_id=5019]
(44) Sort
Input [2]: [demandProfile#3569, demandChannels#3570]
Arguments: [demandProfile#3569 ASC NULLS FIRST], false, 0
(45) Scan parquet
Output [2]: [product#3407, plant#3408]
Batched: true
Location: InMemoryFileIndex [hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-order-qty-opt-service/out/product-plant-list/10000000091/shardId=0_1_10000000091]
ReadSchema: struct<product:string,plant:string>
(46) Filter
Input [3]: [product#3415, plant#3416, demandprofile#3418]
Condition : isnotnull(demandProfile#3418)
(47) BroadcastExchange
Input [3]: [product#3415, plant#3416, demandprofile#3418]
Arguments: HashedRelationBroadcastMode(List(coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[0, string, true], ), isnull(input[0, string, true])),false), [plan_id=5010]
(48) BroadcastHashJoin
Left keys [4]: [coalesce(plant#3408, ), isnull(plant#3408), coalesce(product#3407, ), isnull(product#3407)]
Right keys [4]: [coalesce(plant#3416, ), isnull(plant#3416), coalesce(product#3415, ), isnull(product#3415)]
Join type: Inner
Join condition: None
(49) Project
Output [1]: [demandProfile#3418]
Input [5]: [product#3407, plant#3408, product#3415, plant#3416, demandprofile#3418]
(50) HashAggregate
Input [1]: [demandProfile#3418]
Keys [1]: [demandProfile#3418]
Functions: []
Aggregate Attributes: []
Results [1]: [demandProfile#3418]
(51) Exchange
Input [1]: [demandProfile#3418]
Arguments: hashpartitioning(demandProfile#3418, 37), ENSURE_REQUIREMENTS, [plan_id=5015]
(52) HashAggregate
Input [1]: [demandProfile#3418]
Keys [1]: [demandProfile#3418]
Functions: []
Aggregate Attributes: []
Results [1]: [demandProfile#3418]
(53) Sort
Input [1]: [demandProfile#3418]
Arguments: [demandProfile#3418 ASC NULLS FIRST], false, 0
(54) SortMergeJoin
Left keys [1]: [demandProfile#3569]
Right keys [1]: [demandProfile#3418]
Join type: Inner
Join condition: None
(55) Project
Output [1]: [demandChannels#3570]
Input [3]: [demandProfile#3569, demandChannels#3570, demandProfile#3418]
(56) Generate
Input [1]: [demandChannels#3570]
Arguments: explode(demandChannels#3570), false, [explodedDemandChannels#3582]
(57) Filter
Input [1]: [explodedDemandChannels#3582]
Condition : isnotnull(explodedDemandChannels#3582.name)
(58) Project
Output [1]: [explodedDemandChannels#3582.name AS name#3588]
Input [1]: [explodedDemandChannels#3582]
(59) BroadcastHashJoin
Left keys [1]: [name#3594]
Right keys [1]: [name#3588]
Join type: Inner
Join condition: None
(60) Project
Output [7]: [name#3594, demandStream#3596, forecastCode#3597, distributionChains#3614, customerOrderTypes#3627, customerOrderItemTypeCodes#3639, considerVariance#3603]
Input [8]: [name#3594, demandStream#3596, forecastCode#3597, distributionChains#3614, customerOrderTypes#3627, customerOrderItemTypeCodes#3639, considerVariance#3603, name#3588]
(61) HashAggregate
Input [7]: [name#3594, demandStream#3596, forecastCode#3597, distributionChains#3614, customerOrderTypes#3627, customerOrderItemTypeCodes#3639, considerVariance#3603]
Keys [7]: [name#3594, considerVariance#3603, customerOrderItemTypeCodes#3639, customerOrderTypes#3627, distributionChains#3614, forecastCode#3597, demandStream#3596]
Functions: []
Aggregate Attributes: []
Results [7]: [name#3594, considerVariance#3603, customerOrderItemTypeCodes#3639, customerOrderTypes#3627, distributionChains#3614, forecastCode#3597, demandStream#3596]
(62) Exchange
Input [7]: [name#3594, considerVariance#3603, customerOrderItemTypeCodes#3639, customerOrderTypes#3627, distributionChains#3614, forecastCode#3597, demandStream#3596]
Arguments: hashpartitioning(name#3594, considerVariance#3603, customerOrderItemTypeCodes#3639, customerOrderTypes#3627, distributionChains#3614, forecastCode#3597, demandStream#3596, 37), ENSURE_REQUIREMENTS, [plan_id=5033]
(63) HashAggregate
Input [7]: [name#3594, considerVariance#3603, customerOrderItemTypeCodes#3639, customerOrderTypes#3627, distributionChains#3614, forecastCode#3597, demandStream#3596]
Keys [7]: [name#3594, considerVariance#3603, customerOrderItemTypeCodes#3639, customerOrderTypes#3627, distributionChains#3614, forecastCode#3597, demandStream#3596]
Functions: []
Aggregate Attributes: []
Results: []
(64) AdaptiveSparkPlan
Output: []
Arguments: isFinalPlan=true