digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (6)\n \nduration: 0 ms";
2 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 0 ms<br>peak memory: 64.2 MiB<br>number of output rows: 1<br>number of sort fallback tasks: 0<br>avg hash probes per key: 1"];
}
3 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 251.0 B<br>number of coalesced partitions: 1"];
4 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 1<br>local merged chunks fetched: 0<br>shuffle write time: 0 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 1<br>local bytes read: 244.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 240.0 B<br>local merged bytes read: 0.0 B<br>number of partitions: 37<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 244.0 B"];
subgraph cluster5 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: 13 ms";
6 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 5 ms<br>peak memory: 64.2 MiB<br>number of output rows: 1<br>number of sort fallback tasks: 0<br>avg hash probes per key: 1"];
7 [labelType="html" label="<br><b>Project</b><br><br>"];
8 [labelType="html" label="<b>BroadcastHashJoin</b><br><br>number of output rows: 1"];
15 [labelType="html" label="<br><b>Project</b><br><br>"];
16 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 1"];
17 [labelType="html" label="<b>Generate</b><br><br>number of output rows: 1"];
18 [labelType="html" label="<br><b>Project</b><br><br>"];
19 [labelType="html" label="<b>BroadcastHashJoin</b><br><br>number of output rows: 1"];
26 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 4 ms<br>peak memory: 256.0 KiB<br>number of output rows: 1<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0"];
}
9 [labelType="html" label="<b>BroadcastExchange</b><br><br>time to broadcast: 0 ms<br>time to build: 15 ms<br>time to collect: 369 ms<br>number of output rows: 10<br>data size: 64.0 MiB"];
10 [labelType="html" label="<br><b>Project</b><br><br>"];
subgraph cluster11 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n1.5 s (186 ms, 216 ms, 228 ms (stage 106.0: task 874))";
12 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 10"];
13 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 10<br>number of input batches: 7"];
}
14 [labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 7<br>scan time total (min, med, max (stageId: taskId))<br>1.5 s (182 ms, 213 ms, 225 ms (stage 106.0: task 874))<br>metadata time: 0 ms<br>size of files read: 49.6 KiB<br>number of output rows: 10"];
20 [labelType="html" label="<b>BroadcastExchange</b><br><br>time to broadcast: 0 ms<br>time to build: 6 ms<br>time to collect: 23 ms<br>number of output rows: 10<br>data size: 64.0 MiB"];
21 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1"];
22 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 10<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>6 ms (0 ms, 0 ms, 0 ms (stage 104.0: task 867))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 10<br>local bytes read: 400.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 908.0 B<br>merged fetch fallback count: 0<br>local blocks read: 3<br>remote merged chunks fetched: 0<br>remote blocks read: 7<br>data size total (min, med, max (stageId: taskId))<br>920.0 B (80.0 B, 96.0 B, 104.0 B (stage 104.0: task 858))<br>local merged bytes read: 0.0 B<br>number of partitions: 37<br>remote reqs duration: 3 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>1308.0 B (123.0 B, 132.0 B, 136.0 B (stage 104.0: task 862))"];
subgraph cluster23 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n1.2 s (107 ms, 119 ms, 127 ms (stage 104.0: task 861))";
24 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 10"];
25 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 10"];
}
27 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 2.4 KiB<br>number of coalesced partitions: 1"];
28 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 25<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>16 ms (0 ms, 0 ms, 0 ms (stage 109.0: task 883))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 25<br>local bytes read: 828.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 1472.0 B<br>merged fetch fallback count: 0<br>local blocks read: 9<br>remote merged chunks fetched: 0<br>remote blocks read: 16<br>data size total (min, med, max (stageId: taskId))<br>800.0 B (32.0 B, 32.0 B, 32.0 B (stage 109.0: task 879))<br>local merged bytes read: 0.0 B<br>number of partitions: 37<br>remote reqs duration: 3 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>2.2 KiB (92.0 B, 92.0 B, 92.0 B (stage 109.0: task 879))"];
subgraph cluster29 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n2.3 s (77 ms, 89 ms, 111 ms (stage 109.0: task 882))";
30 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build total (min, med, max (stageId: taskId))<br>2.2 s (74 ms, 86 ms, 104 ms (stage 109.0: task 878))<br>peak memory total (min, med, max (stageId: taskId))<br>6.3 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 109.0: task 879))<br>number of output rows: 25<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0"];
31 [labelType="html" label="<br><b>Project</b><br><br>"];
32 [labelType="html" label="<b>BroadcastHashJoin</b><br><br>number of output rows: 1,174"];
33 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 1,196<br>number of input batches: 25"];
}
34 [labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 25<br>scan time total (min, med, max (stageId: taskId))<br>2.2 s (74 ms, 86 ms, 104 ms (stage 109.0: task 878))<br>metadata time: 0 ms<br>size of files read: 83.2 KiB<br>number of output rows: 1,196"];
35 [labelType="html" label="<b>BroadcastExchange</b><br><br>time to broadcast: 1 ms<br>time to build: 8 ms<br>time to collect: 203 ms<br>number of output rows: 1,174<br>data size: 64.1 MiB"];
subgraph cluster36 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: 177 ms";
37 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 1,174"];
38 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 1,196<br>number of input batches: 1"];
}
39 [labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 1<br>scan time: 176 ms<br>metadata time: 0 ms<br>size of files read: 8.1 KiB<br>number of output rows: 1,196"];
2->0;
3->2;
4->3;
6->4;
7->6;
8->7;
9->8;
10->9;
12->10;
13->12;
14->13;
15->8;
16->15;
17->16;
18->17;
19->18;
20->19;
21->20;
22->21;
24->22;
25->24;
26->19;
27->26;
28->27;
30->28;
31->30;
32->31;
33->32;
34->33;
35->32;
37->35;
38->37;
39->38;
}
40
AdaptiveSparkPlan isFinalPlan=true
HashAggregate(keys=[name#2984, considerVariance#2993, customerOrderItemTypeCodes#3029, customerOrderTypes#3017, distributionChains#3004, forecastCode#2987, demandStream#2986], functions=[])
WholeStageCodegen (6)
AQEShuffleRead coalesced
Exchange hashpartitioning(name#2984, considerVariance#2993, customerOrderItemTypeCodes#3029, customerOrderTypes#3017, distributionChains#3004, forecastCode#2987, demandStream#2986, 37), ENSURE_REQUIREMENTS, [plan_id=4405]
HashAggregate(keys=[name#2984, considerVariance#2993, customerOrderItemTypeCodes#3029, customerOrderTypes#3017, distributionChains#3004, forecastCode#2987, demandStream#2986], functions=[])
Project [name#2984, demandStream#2986, forecastCode#2987, distributionChains#3004, customerOrderTypes#3017, customerOrderItemTypeCodes#3029, considerVariance#2993]
BroadcastHashJoin [name#2984], [name#2978], Inner, BuildLeft, false
Project [explodedDemandChannels#2972.name AS name#2978]
Filter isnotnull(explodedDemandChannels#2972.name)
Generate explode(demandChannels#2960), false, [explodedDemandChannels#2972]
Project [demandChannels#2960]
BroadcastHashJoin [demandProfile#2959], [demandProfile#2808], Inner, BuildLeft, false
HashAggregate(keys=[demandProfile#2808], functions=[])
WholeStageCodegen (5)
BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3970]
Project [name#2984, demandStream#2986, forecastCode#2987, transform(distributionChains#2989, lambdafunction(named_struct(internalUUID, lambda chain#3005.internalUUID, salesOrganizationUUID, lambda chain#3005.salesOrganization.internalRefUUID, distributionChannelUUID, lambda chain#3005.distributionChannel.internalRefUUID), lambda chain#3005, false)) AS distributionChains#3004, transform(customerOrderTypes#2990, lambdafunction(named_struct(internalUUID, lambda documentType#3018.internalRefUUID), lambda documentType#3018, false)) AS customerOrderTypes#3017, transform(customerOrderItemTypeCodes#2991, lambdafunction(named_struct(customerOrderItemTypeCodeUUID, lambda itemType#3030.internalRefUUID), lambda itemType#3030, false)) AS customerOrderItemTypeCodes#3029, considerVariance#2993]
Filter isnotnull(name#2984)
ColumnarToRow
WholeStageCodegen (1)
FileScan parquet [name#2984,demandStream#2986,forecastCode#2987,distributionChains#2989,customerOrderTypes#2990,customerOrderItemTypeCodes#2991,considerVariance#2993] Batched: true, DataFilters: [isnotnull(name#2984)], Format: Parquet, Location: PreparedDeltaFileIndex(1 paths)[hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20...., PartitionFilters: [], PushedFilters: [IsNotNull(name)], ReadSchema: struct<name:string,demandStream:int,forecastCode:string,distributionChains:array<struct<internalU...
BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false]),false), [plan_id=4110]
AQEShuffleRead local
Exchange hashpartitioning(demandProfile#2959, 37), ENSURE_REQUIREMENTS, [plan_id=3981]
Filter (isnotnull(demandProfile#2959) AND ((size(demandChannels#2960, true) > 0) AND isnotnull(demandChannels#2960)))
Scan ExistingRDD[demandProfile#2959,demandChannels#2960]
WholeStageCodegen (2)
AQEShuffleRead coalesced
Exchange hashpartitioning(demandProfile#2808, 37), ENSURE_REQUIREMENTS, [plan_id=4270]
HashAggregate(keys=[demandProfile#2808], functions=[])
Project [demandProfile#2808]
BroadcastHashJoin [coalesce(plant#2798, ), isnull(plant#2798), coalesce(product#2797, ), isnull(product#2797)], [coalesce(plant#2806, ), isnull(plant#2806), coalesce(product#2805, ), isnull(product#2805)], Inner, BuildRight, false
ColumnarToRow
WholeStageCodegen (4)
FileScan parquet [product#2797,plant#2798] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20.hanac..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<product:string,plant:string>
BroadcastExchange HashedRelationBroadcastMode(List(coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[0, string, true], ), isnull(input[0, string, true])),false), [plan_id=3994]
Filter isnotnull(demandProfile#2808)
ColumnarToRow
WholeStageCodegen (3)
FileScan parquet [product#2805,plant#2806,demandprofile#2808] Batched: true, DataFilters: [isnotnull(demandprofile#2808)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20.hanac..., PartitionFilters: [], PushedFilters: [IsNotNull(demandprofile)], ReadSchema: struct<product:string,plant:string,demandprofile:string>
== Physical Plan ==
AdaptiveSparkPlan (64)
+- == Final Plan ==
* HashAggregate (39)
+- AQEShuffleRead (38)
+- ShuffleQueryStage (37), Statistics(sizeInBytes=240.0 B, rowCount=1)
+- Exchange (36)
+- * HashAggregate (35)
+- * Project (34)
+- * BroadcastHashJoin Inner BuildLeft (33)
:- BroadcastQueryStage (6), Statistics(sizeInBytes=64.0 MiB, rowCount=10)
: +- BroadcastExchange (5)
: +- Project (4)
: +- * Filter (3)
: +- * ColumnarToRow (2)
: +- Scan parquet (1)
+- * Project (32)
+- * Filter (31)
+- * Generate (30)
+- * Project (29)
+- * BroadcastHashJoin Inner BuildLeft (28)
:- BroadcastQueryStage (13), Statistics(sizeInBytes=64.0 MiB, rowCount=10)
: +- BroadcastExchange (12)
: +- AQEShuffleRead (11)
: +- ShuffleQueryStage (10), Statistics(sizeInBytes=920.0 B, rowCount=10)
: +- Exchange (9)
: +- * Filter (8)
: +- * Scan ExistingRDD (7)
+- * HashAggregate (27)
+- AQEShuffleRead (26)
+- ShuffleQueryStage (25), Statistics(sizeInBytes=800.0 B, rowCount=25)
+- Exchange (24)
+- * HashAggregate (23)
+- * Project (22)
+- * BroadcastHashJoin Inner BuildRight (21)
:- * ColumnarToRow (15)
: +- Scan parquet (14)
+- BroadcastQueryStage (20), Statistics(sizeInBytes=64.1 MiB, rowCount=1.17E+3)
+- BroadcastExchange (19)
+- * Filter (18)
+- * ColumnarToRow (17)
+- Scan parquet (16)
+- == Initial Plan ==
HashAggregate (63)
+- Exchange (62)
+- HashAggregate (61)
+- Project (60)
+- BroadcastHashJoin Inner BuildLeft (59)
:- BroadcastExchange (42)
: +- Project (41)
: +- Filter (40)
: +- Scan parquet (1)
+- Project (58)
+- Filter (57)
+- Generate (56)
+- Project (55)
+- SortMergeJoin Inner (54)
:- Sort (44)
: +- Exchange (43)
: +- Filter (8)
: +- Scan ExistingRDD (7)
+- Sort (53)
+- HashAggregate (52)
+- Exchange (51)
+- HashAggregate (50)
+- Project (49)
+- BroadcastHashJoin Inner BuildRight (48)
:- Scan parquet (45)
+- BroadcastExchange (47)
+- Filter (46)
+- Scan parquet (16)
(1) Scan parquet
Output [7]: [name#2984, demandStream#2986, forecastCode#2987, distributionChains#2989, customerOrderTypes#2990, customerOrderItemTypeCodes#2991, considerVariance#2993]
Batched: true
Location: PreparedDeltaFileIndex [hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-dl-stream-service/prp/demand-channel]
PushedFilters: [IsNotNull(name)]
ReadSchema: struct<name:string,demandStream:int,forecastCode:string,distributionChains:array<struct<internalUUID:string,salesOrganization:struct<internalRefUUID:string,id:string>,distributionChannel:struct<internalRefUUID:string,id:string>>>,customerOrderTypes:array<struct<id:string,internalRefUUID:string>>,customerOrderItemTypeCodes:array<struct<id:string,internalRefUUID:string>>,considerVariance:boolean>
(2) ColumnarToRow [codegen id : 1]
Input [7]: [name#2984, demandStream#2986, forecastCode#2987, distributionChains#2989, customerOrderTypes#2990, customerOrderItemTypeCodes#2991, considerVariance#2993]
(3) Filter [codegen id : 1]
Input [7]: [name#2984, demandStream#2986, forecastCode#2987, distributionChains#2989, customerOrderTypes#2990, customerOrderItemTypeCodes#2991, considerVariance#2993]
Condition : isnotnull(name#2984)
(4) Project
Output [7]: [name#2984, demandStream#2986, forecastCode#2987, transform(distributionChains#2989, lambdafunction(named_struct(internalUUID, lambda chain#3005.internalUUID, salesOrganizationUUID, lambda chain#3005.salesOrganization.internalRefUUID, distributionChannelUUID, lambda chain#3005.distributionChannel.internalRefUUID), lambda chain#3005, false)) AS distributionChains#3004, transform(customerOrderTypes#2990, lambdafunction(named_struct(internalUUID, lambda documentType#3018.internalRefUUID), lambda documentType#3018, false)) AS customerOrderTypes#3017, transform(customerOrderItemTypeCodes#2991, lambdafunction(named_struct(customerOrderItemTypeCodeUUID, lambda itemType#3030.internalRefUUID), lambda itemType#3030, false)) AS customerOrderItemTypeCodes#3029, considerVariance#2993]
Input [7]: [name#2984, demandStream#2986, forecastCode#2987, distributionChains#2989, customerOrderTypes#2990, customerOrderItemTypeCodes#2991, considerVariance#2993]
(5) BroadcastExchange
Input [7]: [name#2984, demandStream#2986, forecastCode#2987, distributionChains#3004, customerOrderTypes#3017, customerOrderItemTypeCodes#3029, considerVariance#2993]
Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3970]
(6) BroadcastQueryStage
Output [7]: [name#2984, demandStream#2986, forecastCode#2987, distributionChains#3004, customerOrderTypes#3017, customerOrderItemTypeCodes#3029, considerVariance#2993]
Arguments: 0
(7) Scan ExistingRDD [codegen id : 2]
Output [2]: [demandProfile#2959, demandChannels#2960]
Arguments: [demandProfile#2959, demandChannels#2960], MapPartitionsRDD[270] at createDataFrame at DemandProfileDataReaderImpl.java:60, ExistingRDD, UnknownPartitioning(0)
(8) Filter [codegen id : 2]
Input [2]: [demandProfile#2959, demandChannels#2960]
Condition : (isnotnull(demandProfile#2959) AND ((size(demandChannels#2960, true) > 0) AND isnotnull(demandChannels#2960)))
(9) Exchange
Input [2]: [demandProfile#2959, demandChannels#2960]
Arguments: hashpartitioning(demandProfile#2959, 37), ENSURE_REQUIREMENTS, [plan_id=3981]
(10) ShuffleQueryStage
Output [2]: [demandProfile#2959, demandChannels#2960]
Arguments: 1
(11) AQEShuffleRead
Input [2]: [demandProfile#2959, demandChannels#2960]
Arguments: local
(12) BroadcastExchange
Input [2]: [demandProfile#2959, demandChannels#2960]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false]),false), [plan_id=4110]
(13) BroadcastQueryStage
Output [2]: [demandProfile#2959, demandChannels#2960]
Arguments: 3
(14) Scan parquet
Output [2]: [product#2797, plant#2798]
Batched: true
Location: InMemoryFileIndex [hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-order-qty-opt-service/out/product-plant-list/10000000091/shardId=0_1_10000000091]
ReadSchema: struct<product:string,plant:string>
(15) ColumnarToRow [codegen id : 4]
Input [2]: [product#2797, plant#2798]
(16) Scan parquet
Output [3]: [product#2805, plant#2806, demandprofile#2808]
Batched: true
Location: InMemoryFileIndex [hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-param-determination-service/out/10000000091/0_1_10000000091/demandprofile]
PushedFilters: [IsNotNull(demandprofile)]
ReadSchema: struct<product:string,plant:string,demandprofile:string>
(17) ColumnarToRow [codegen id : 3]
Input [3]: [product#2805, plant#2806, demandprofile#2808]
(18) Filter [codegen id : 3]
Input [3]: [product#2805, plant#2806, demandprofile#2808]
Condition : isnotnull(demandProfile#2808)
(19) BroadcastExchange
Input [3]: [product#2805, plant#2806, demandprofile#2808]
Arguments: HashedRelationBroadcastMode(List(coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[0, string, true], ), isnull(input[0, string, true])),false), [plan_id=3994]
(20) BroadcastQueryStage
Output [3]: [product#2805, plant#2806, demandprofile#2808]
Arguments: 2
(21) BroadcastHashJoin [codegen id : 4]
Left keys [4]: [coalesce(plant#2798, ), isnull(plant#2798), coalesce(product#2797, ), isnull(product#2797)]
Right keys [4]: [coalesce(plant#2806, ), isnull(plant#2806), coalesce(product#2805, ), isnull(product#2805)]
Join type: Inner
Join condition: None
(22) Project [codegen id : 4]
Output [1]: [demandProfile#2808]
Input [5]: [product#2797, plant#2798, product#2805, plant#2806, demandprofile#2808]
(23) HashAggregate [codegen id : 4]
Input [1]: [demandProfile#2808]
Keys [1]: [demandProfile#2808]
Functions: []
Aggregate Attributes: []
Results [1]: [demandProfile#2808]
(24) Exchange
Input [1]: [demandProfile#2808]
Arguments: hashpartitioning(demandProfile#2808, 37), ENSURE_REQUIREMENTS, [plan_id=4270]
(25) ShuffleQueryStage
Output [1]: [demandProfile#2808]
Arguments: 4
(26) AQEShuffleRead
Input [1]: [demandProfile#2808]
Arguments: coalesced
(27) HashAggregate
Input [1]: [demandProfile#2808]
Keys [1]: [demandProfile#2808]
Functions: []
Aggregate Attributes: []
Results [1]: [demandProfile#2808]
(28) BroadcastHashJoin
Left keys [1]: [demandProfile#2959]
Right keys [1]: [demandProfile#2808]
Join type: Inner
Join condition: None
(29) Project
Output [1]: [demandChannels#2960]
Input [3]: [demandProfile#2959, demandChannels#2960, demandProfile#2808]
(30) Generate
Input [1]: [demandChannels#2960]
Arguments: explode(demandChannels#2960), false, [explodedDemandChannels#2972]
(31) Filter
Input [1]: [explodedDemandChannels#2972]
Condition : isnotnull(explodedDemandChannels#2972.name)
(32) Project
Output [1]: [explodedDemandChannels#2972.name AS name#2978]
Input [1]: [explodedDemandChannels#2972]
(33) BroadcastHashJoin [codegen id : 5]
Left keys [1]: [name#2984]
Right keys [1]: [name#2978]
Join type: Inner
Join condition: None
(34) Project [codegen id : 5]
Output [7]: [name#2984, demandStream#2986, forecastCode#2987, distributionChains#3004, customerOrderTypes#3017, customerOrderItemTypeCodes#3029, considerVariance#2993]
Input [8]: [name#2984, demandStream#2986, forecastCode#2987, distributionChains#3004, customerOrderTypes#3017, customerOrderItemTypeCodes#3029, considerVariance#2993, name#2978]
(35) HashAggregate [codegen id : 5]
Input [7]: [name#2984, demandStream#2986, forecastCode#2987, distributionChains#3004, customerOrderTypes#3017, customerOrderItemTypeCodes#3029, considerVariance#2993]
Keys [7]: [name#2984, considerVariance#2993, customerOrderItemTypeCodes#3029, customerOrderTypes#3017, distributionChains#3004, forecastCode#2987, demandStream#2986]
Functions: []
Aggregate Attributes: []
Results [7]: [name#2984, considerVariance#2993, customerOrderItemTypeCodes#3029, customerOrderTypes#3017, distributionChains#3004, forecastCode#2987, demandStream#2986]
(36) Exchange
Input [7]: [name#2984, considerVariance#2993, customerOrderItemTypeCodes#3029, customerOrderTypes#3017, distributionChains#3004, forecastCode#2987, demandStream#2986]
Arguments: hashpartitioning(name#2984, considerVariance#2993, customerOrderItemTypeCodes#3029, customerOrderTypes#3017, distributionChains#3004, forecastCode#2987, demandStream#2986, 37), ENSURE_REQUIREMENTS, [plan_id=4405]
(37) ShuffleQueryStage
Output [7]: [name#2984, considerVariance#2993, customerOrderItemTypeCodes#3029, customerOrderTypes#3017, distributionChains#3004, forecastCode#2987, demandStream#2986]
Arguments: 5
(38) AQEShuffleRead
Input [7]: [name#2984, considerVariance#2993, customerOrderItemTypeCodes#3029, customerOrderTypes#3017, distributionChains#3004, forecastCode#2987, demandStream#2986]
Arguments: coalesced
(39) HashAggregate [codegen id : 6]
Input [7]: [name#2984, considerVariance#2993, customerOrderItemTypeCodes#3029, customerOrderTypes#3017, distributionChains#3004, forecastCode#2987, demandStream#2986]
Keys [7]: [name#2984, considerVariance#2993, customerOrderItemTypeCodes#3029, customerOrderTypes#3017, distributionChains#3004, forecastCode#2987, demandStream#2986]
Functions: []
Aggregate Attributes: []
Results: []
(40) Filter
Input [7]: [name#2984, demandStream#2986, forecastCode#2987, distributionChains#2989, customerOrderTypes#2990, customerOrderItemTypeCodes#2991, considerVariance#2993]
Condition : isnotnull(name#2984)
(41) Project
Output [7]: [name#2984, demandStream#2986, forecastCode#2987, transform(distributionChains#2989, lambdafunction(named_struct(internalUUID, lambda chain#3005.internalUUID, salesOrganizationUUID, lambda chain#3005.salesOrganization.internalRefUUID, distributionChannelUUID, lambda chain#3005.distributionChannel.internalRefUUID), lambda chain#3005, false)) AS distributionChains#3004, transform(customerOrderTypes#2990, lambdafunction(named_struct(internalUUID, lambda documentType#3018.internalRefUUID), lambda documentType#3018, false)) AS customerOrderTypes#3017, transform(customerOrderItemTypeCodes#2991, lambdafunction(named_struct(customerOrderItemTypeCodeUUID, lambda itemType#3030.internalRefUUID), lambda itemType#3030, false)) AS customerOrderItemTypeCodes#3029, considerVariance#2993]
Input [7]: [name#2984, demandStream#2986, forecastCode#2987, distributionChains#2989, customerOrderTypes#2990, customerOrderItemTypeCodes#2991, considerVariance#2993]
(42) BroadcastExchange
Input [7]: [name#2984, demandStream#2986, forecastCode#2987, distributionChains#3004, customerOrderTypes#3017, customerOrderItemTypeCodes#3029, considerVariance#2993]
Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3906]
(43) Exchange
Input [2]: [demandProfile#2959, demandChannels#2960]
Arguments: hashpartitioning(demandProfile#2959, 37), ENSURE_REQUIREMENTS, [plan_id=3897]
(44) Sort
Input [2]: [demandProfile#2959, demandChannels#2960]
Arguments: [demandProfile#2959 ASC NULLS FIRST], false, 0
(45) Scan parquet
Output [2]: [product#2797, plant#2798]
Batched: true
Location: InMemoryFileIndex [hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-order-qty-opt-service/out/product-plant-list/10000000091/shardId=0_1_10000000091]
ReadSchema: struct<product:string,plant:string>
(46) Filter
Input [3]: [product#2805, plant#2806, demandprofile#2808]
Condition : isnotnull(demandProfile#2808)
(47) BroadcastExchange
Input [3]: [product#2805, plant#2806, demandprofile#2808]
Arguments: HashedRelationBroadcastMode(List(coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[0, string, true], ), isnull(input[0, string, true])),false), [plan_id=3888]
(48) BroadcastHashJoin
Left keys [4]: [coalesce(plant#2798, ), isnull(plant#2798), coalesce(product#2797, ), isnull(product#2797)]
Right keys [4]: [coalesce(plant#2806, ), isnull(plant#2806), coalesce(product#2805, ), isnull(product#2805)]
Join type: Inner
Join condition: None
(49) Project
Output [1]: [demandProfile#2808]
Input [5]: [product#2797, plant#2798, product#2805, plant#2806, demandprofile#2808]
(50) HashAggregate
Input [1]: [demandProfile#2808]
Keys [1]: [demandProfile#2808]
Functions: []
Aggregate Attributes: []
Results [1]: [demandProfile#2808]
(51) Exchange
Input [1]: [demandProfile#2808]
Arguments: hashpartitioning(demandProfile#2808, 37), ENSURE_REQUIREMENTS, [plan_id=3893]
(52) HashAggregate
Input [1]: [demandProfile#2808]
Keys [1]: [demandProfile#2808]
Functions: []
Aggregate Attributes: []
Results [1]: [demandProfile#2808]
(53) Sort
Input [1]: [demandProfile#2808]
Arguments: [demandProfile#2808 ASC NULLS FIRST], false, 0
(54) SortMergeJoin
Left keys [1]: [demandProfile#2959]
Right keys [1]: [demandProfile#2808]
Join type: Inner
Join condition: None
(55) Project
Output [1]: [demandChannels#2960]
Input [3]: [demandProfile#2959, demandChannels#2960, demandProfile#2808]
(56) Generate
Input [1]: [demandChannels#2960]
Arguments: explode(demandChannels#2960), false, [explodedDemandChannels#2972]
(57) Filter
Input [1]: [explodedDemandChannels#2972]
Condition : isnotnull(explodedDemandChannels#2972.name)
(58) Project
Output [1]: [explodedDemandChannels#2972.name AS name#2978]
Input [1]: [explodedDemandChannels#2972]
(59) BroadcastHashJoin
Left keys [1]: [name#2984]
Right keys [1]: [name#2978]
Join type: Inner
Join condition: None
(60) Project
Output [7]: [name#2984, demandStream#2986, forecastCode#2987, distributionChains#3004, customerOrderTypes#3017, customerOrderItemTypeCodes#3029, considerVariance#2993]
Input [8]: [name#2984, demandStream#2986, forecastCode#2987, distributionChains#3004, customerOrderTypes#3017, customerOrderItemTypeCodes#3029, considerVariance#2993, name#2978]
(61) HashAggregate
Input [7]: [name#2984, demandStream#2986, forecastCode#2987, distributionChains#3004, customerOrderTypes#3017, customerOrderItemTypeCodes#3029, considerVariance#2993]
Keys [7]: [name#2984, considerVariance#2993, customerOrderItemTypeCodes#3029, customerOrderTypes#3017, distributionChains#3004, forecastCode#2987, demandStream#2986]
Functions: []
Aggregate Attributes: []
Results [7]: [name#2984, considerVariance#2993, customerOrderItemTypeCodes#3029, customerOrderTypes#3017, distributionChains#3004, forecastCode#2987, demandStream#2986]
(62) Exchange
Input [7]: [name#2984, considerVariance#2993, customerOrderItemTypeCodes#3029, customerOrderTypes#3017, distributionChains#3004, forecastCode#2987, demandStream#2986]
Arguments: hashpartitioning(name#2984, considerVariance#2993, customerOrderItemTypeCodes#3029, customerOrderTypes#3017, distributionChains#3004, forecastCode#2987, demandStream#2986, 37), ENSURE_REQUIREMENTS, [plan_id=3911]
(63) HashAggregate
Input [7]: [name#2984, considerVariance#2993, customerOrderItemTypeCodes#3029, customerOrderTypes#3017, distributionChains#3004, forecastCode#2987, demandStream#2986]
Keys [7]: [name#2984, considerVariance#2993, customerOrderItemTypeCodes#3029, customerOrderTypes#3017, distributionChains#3004, forecastCode#2987, demandStream#2986]
Functions: []
Aggregate Attributes: []
Results: []
(64) AdaptiveSparkPlan
Output: []
Arguments: isFinalPlan=true