digraph G {
0 [labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>"];
subgraph cluster1 {
isCluster="true";
label="WholeStageCodegen (6)\n \nduration: 0 ms";
2 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 0 ms<br>peak memory: 64.2 MiB<br>number of output rows: 1<br>number of sort fallback tasks: 0<br>avg hash probes per key: 1"];
}
3 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 251.0 B<br>number of coalesced partitions: 1"];
4 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 1<br>local merged chunks fetched: 0<br>shuffle write time: 0 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 1<br>local bytes read: 244.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 1<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 240.0 B<br>local merged bytes read: 0.0 B<br>number of partitions: 37<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 244.0 B"];
subgraph cluster5 {
isCluster="true";
label="WholeStageCodegen (5)\n \nduration: 32 ms";
6 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 7 ms<br>peak memory: 64.2 MiB<br>number of output rows: 1<br>number of sort fallback tasks: 0<br>avg hash probes per key: 1"];
7 [labelType="html" label="<br><b>Project</b><br><br>"];
8 [labelType="html" label="<b>BroadcastHashJoin</b><br><br>number of output rows: 1"];
15 [labelType="html" label="<br><b>Project</b><br><br>"];
16 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 1"];
17 [labelType="html" label="<b>Generate</b><br><br>number of output rows: 1"];
18 [labelType="html" label="<br><b>Project</b><br><br>"];
19 [labelType="html" label="<b>BroadcastHashJoin</b><br><br>number of output rows: 1"];
26 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 5 ms<br>peak memory: 256.0 KiB<br>number of output rows: 1<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0"];
}
9 [labelType="html" label="<b>BroadcastExchange</b><br><br>time to broadcast: 0 ms<br>time to build: 16 ms<br>time to collect: 393 ms<br>number of output rows: 10<br>data size: 64.0 MiB"];
10 [labelType="html" label="<br><b>Project</b><br><br>"];
subgraph cluster11 {
isCluster="true";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n1.5 s (188 ms, 210 ms, 245 ms (stage 50.0: task 448))";
12 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 10"];
13 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 10<br>number of input batches: 7"];
}
14 [labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 7<br>scan time total (min, med, max (stageId: taskId))<br>1.5 s (184 ms, 207 ms, 242 ms (stage 50.0: task 448))<br>metadata time: 0 ms<br>size of files read: 49.6 KiB<br>number of output rows: 10"];
20 [labelType="html" label="<b>BroadcastExchange</b><br><br>time to broadcast: 0 ms<br>time to build: 18 ms<br>time to collect: 27 ms<br>number of output rows: 10<br>data size: 64.0 MiB"];
21 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1"];
22 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 10<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>8 ms (0 ms, 0 ms, 1 ms (stage 48.0: task 434))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 10<br>local bytes read: 520.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 788.0 B<br>merged fetch fallback count: 0<br>local blocks read: 4<br>remote merged chunks fetched: 0<br>remote blocks read: 6<br>data size total (min, med, max (stageId: taskId))<br>920.0 B (80.0 B, 96.0 B, 104.0 B (stage 48.0: task 432))<br>local merged bytes read: 0.0 B<br>number of partitions: 37<br>remote reqs duration: 2 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>1308.0 B (123.0 B, 132.0 B, 136.0 B (stage 48.0: task 436))"];
subgraph cluster23 {
isCluster="true";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n1.2 s (112 ms, 116 ms, 145 ms (stage 48.0: task 433))";
24 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 10"];
25 [labelType="html" label="<b>Scan ExistingRDD</b><br><br>number of output rows: 10"];
}
27 [labelType="html" label="<b>AQEShuffleRead</b><br><br>number of partitions: 1<br>partition data size: 2.4 KiB<br>number of coalesced partitions: 1"];
28 [labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 25<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>16 ms (0 ms, 0 ms, 0 ms (stage 53.0: task 466))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 25<br>local bytes read: 828.0 B<br>fetch wait time: 0 ms<br>remote bytes read: 1472.0 B<br>merged fetch fallback count: 0<br>local blocks read: 9<br>remote merged chunks fetched: 0<br>remote blocks read: 16<br>data size total (min, med, max (stageId: taskId))<br>800.0 B (32.0 B, 32.0 B, 32.0 B (stage 53.0: task 456))<br>local merged bytes read: 0.0 B<br>number of partitions: 37<br>remote reqs duration: 4 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>2.2 KiB (92.0 B, 92.0 B, 92.0 B (stage 53.0: task 456))"];
subgraph cluster29 {
isCluster="true";
label="WholeStageCodegen (4)\n \nduration: total (min, med, max (stageId: taskId))\n2.5 s (82 ms, 97 ms, 139 ms (stage 53.0: task 451))";
30 [labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build total (min, med, max (stageId: taskId))<br>2.3 s (79 ms, 91 ms, 129 ms (stage 53.0: task 451))<br>peak memory total (min, med, max (stageId: taskId))<br>6.3 MiB (256.0 KiB, 256.0 KiB, 256.0 KiB (stage 53.0: task 456))<br>number of output rows: 25<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0"];
31 [labelType="html" label="<br><b>Project</b><br><br>"];
32 [labelType="html" label="<b>BroadcastHashJoin</b><br><br>number of output rows: 1,174"];
33 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 1,196<br>number of input batches: 25"];
}
34 [labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 25<br>scan time total (min, med, max (stageId: taskId))<br>2.3 s (79 ms, 91 ms, 128 ms (stage 53.0: task 451))<br>metadata time: 0 ms<br>size of files read: 83.2 KiB<br>number of output rows: 1,196"];
35 [labelType="html" label="<b>BroadcastExchange</b><br><br>time to broadcast: 1 ms<br>time to build: 10 ms<br>time to collect: 222 ms<br>number of output rows: 1,174<br>data size: 64.1 MiB"];
subgraph cluster36 {
isCluster="true";
label="WholeStageCodegen (3)\n \nduration: 173 ms";
37 [labelType="html" label="<b>Filter</b><br><br>number of output rows: 1,174"];
38 [labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 1,196<br>number of input batches: 1"];
}
39 [labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 1<br>scan time: 170 ms<br>metadata time: 0 ms<br>size of files read: 8.1 KiB<br>number of output rows: 1,196"];
2->0;
3->2;
4->3;
6->4;
7->6;
8->7;
9->8;
10->9;
12->10;
13->12;
14->13;
15->8;
16->15;
17->16;
18->17;
19->18;
20->19;
21->20;
22->21;
24->22;
25->24;
26->19;
27->26;
28->27;
30->28;
31->30;
32->31;
33->32;
34->33;
35->32;
37->35;
38->37;
39->38;
}
40
AdaptiveSparkPlan isFinalPlan=true
HashAggregate(keys=[name#1764, considerVariance#1773, customerOrderItemTypeCodes#1809, customerOrderTypes#1797, distributionChains#1784, forecastCode#1767, demandStream#1766], functions=[])
WholeStageCodegen (6)
AQEShuffleRead coalesced
Exchange hashpartitioning(name#1764, considerVariance#1773, customerOrderItemTypeCodes#1809, customerOrderTypes#1797, distributionChains#1784, forecastCode#1767, demandStream#1766, 37), ENSURE_REQUIREMENTS, [plan_id=2181]
HashAggregate(keys=[name#1764, considerVariance#1773, customerOrderItemTypeCodes#1809, customerOrderTypes#1797, distributionChains#1784, forecastCode#1767, demandStream#1766], functions=[])
Project [name#1764, demandStream#1766, forecastCode#1767, distributionChains#1784, customerOrderTypes#1797, customerOrderItemTypeCodes#1809, considerVariance#1773]
BroadcastHashJoin [name#1764], [name#1758], Inner, BuildLeft, false
Project [explodedDemandChannels#1752.name AS name#1758]
Filter isnotnull(explodedDemandChannels#1752.name)
Generate explode(demandChannels#1740), false, [explodedDemandChannels#1752]
Project [demandChannels#1740]
BroadcastHashJoin [demandProfile#1739], [demandProfile#1588], Inner, BuildLeft, false
HashAggregate(keys=[demandProfile#1588], functions=[])
WholeStageCodegen (5)
BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=1746]
Project [name#1764, demandStream#1766, forecastCode#1767, transform(distributionChains#1769, lambdafunction(named_struct(internalUUID, lambda chain#1785.internalUUID, salesOrganizationUUID, lambda chain#1785.salesOrganization.internalRefUUID, distributionChannelUUID, lambda chain#1785.distributionChannel.internalRefUUID), lambda chain#1785, false)) AS distributionChains#1784, transform(customerOrderTypes#1770, lambdafunction(named_struct(internalUUID, lambda documentType#1798.internalRefUUID), lambda documentType#1798, false)) AS customerOrderTypes#1797, transform(customerOrderItemTypeCodes#1771, lambdafunction(named_struct(customerOrderItemTypeCodeUUID, lambda itemType#1810.internalRefUUID), lambda itemType#1810, false)) AS customerOrderItemTypeCodes#1809, considerVariance#1773]
Filter isnotnull(name#1764)
ColumnarToRow
WholeStageCodegen (1)
FileScan parquet [name#1764,demandStream#1766,forecastCode#1767,distributionChains#1769,customerOrderTypes#1770,customerOrderItemTypeCodes#1771,considerVariance#1773] Batched: true, DataFilters: [isnotnull(name#1764)], Format: Parquet, Location: PreparedDeltaFileIndex(1 paths)[hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20...., PartitionFilters: [], PushedFilters: [IsNotNull(name)], ReadSchema: struct<name:string,demandStream:int,forecastCode:string,distributionChains:array<struct<internalU...
BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false]),false), [plan_id=1886]
AQEShuffleRead local
Exchange hashpartitioning(demandProfile#1739, 37), ENSURE_REQUIREMENTS, [plan_id=1757]
Filter (isnotnull(demandProfile#1739) AND ((size(demandChannels#1740, true) > 0) AND isnotnull(demandChannels#1740)))
Scan ExistingRDD[demandProfile#1739,demandChannels#1740]
WholeStageCodegen (2)
AQEShuffleRead coalesced
Exchange hashpartitioning(demandProfile#1588, 37), ENSURE_REQUIREMENTS, [plan_id=2046]
HashAggregate(keys=[demandProfile#1588], functions=[])
Project [demandProfile#1588]
BroadcastHashJoin [coalesce(plant#1578, ), isnull(plant#1578), coalesce(product#1577, ), isnull(product#1577)], [coalesce(plant#1586, ), isnull(plant#1586), coalesce(product#1585, ), isnull(product#1585)], Inner, BuildRight, false
ColumnarToRow
WholeStageCodegen (4)
FileScan parquet [product#1577,plant#1578] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20.hanac..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<product:string,plant:string>
BroadcastExchange HashedRelationBroadcastMode(List(coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[0, string, true], ), isnull(input[0, string, true])),false), [plan_id=1770]
Filter isnotnull(demandProfile#1588)
ColumnarToRow
WholeStageCodegen (3)
FileScan parquet [product#1585,plant#1586,demandprofile#1588] Batched: true, DataFilters: [isnotnull(demandprofile#1588)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20.hanac..., PartitionFilters: [], PushedFilters: [IsNotNull(demandprofile)], ReadSchema: struct<product:string,plant:string,demandprofile:string>
== Physical Plan ==
AdaptiveSparkPlan (64)
+- == Final Plan ==
* HashAggregate (39)
+- AQEShuffleRead (38)
+- ShuffleQueryStage (37), Statistics(sizeInBytes=240.0 B, rowCount=1)
+- Exchange (36)
+- * HashAggregate (35)
+- * Project (34)
+- * BroadcastHashJoin Inner BuildLeft (33)
:- BroadcastQueryStage (6), Statistics(sizeInBytes=64.0 MiB, rowCount=10)
: +- BroadcastExchange (5)
: +- Project (4)
: +- * Filter (3)
: +- * ColumnarToRow (2)
: +- Scan parquet (1)
+- * Project (32)
+- * Filter (31)
+- * Generate (30)
+- * Project (29)
+- * BroadcastHashJoin Inner BuildLeft (28)
:- BroadcastQueryStage (13), Statistics(sizeInBytes=64.0 MiB, rowCount=10)
: +- BroadcastExchange (12)
: +- AQEShuffleRead (11)
: +- ShuffleQueryStage (10), Statistics(sizeInBytes=920.0 B, rowCount=10)
: +- Exchange (9)
: +- * Filter (8)
: +- * Scan ExistingRDD (7)
+- * HashAggregate (27)
+- AQEShuffleRead (26)
+- ShuffleQueryStage (25), Statistics(sizeInBytes=800.0 B, rowCount=25)
+- Exchange (24)
+- * HashAggregate (23)
+- * Project (22)
+- * BroadcastHashJoin Inner BuildRight (21)
:- * ColumnarToRow (15)
: +- Scan parquet (14)
+- BroadcastQueryStage (20), Statistics(sizeInBytes=64.1 MiB, rowCount=1.17E+3)
+- BroadcastExchange (19)
+- * Filter (18)
+- * ColumnarToRow (17)
+- Scan parquet (16)
+- == Initial Plan ==
HashAggregate (63)
+- Exchange (62)
+- HashAggregate (61)
+- Project (60)
+- BroadcastHashJoin Inner BuildLeft (59)
:- BroadcastExchange (42)
: +- Project (41)
: +- Filter (40)
: +- Scan parquet (1)
+- Project (58)
+- Filter (57)
+- Generate (56)
+- Project (55)
+- SortMergeJoin Inner (54)
:- Sort (44)
: +- Exchange (43)
: +- Filter (8)
: +- Scan ExistingRDD (7)
+- Sort (53)
+- HashAggregate (52)
+- Exchange (51)
+- HashAggregate (50)
+- Project (49)
+- BroadcastHashJoin Inner BuildRight (48)
:- Scan parquet (45)
+- BroadcastExchange (47)
+- Filter (46)
+- Scan parquet (16)
(1) Scan parquet
Output [7]: [name#1764, demandStream#1766, forecastCode#1767, distributionChains#1769, customerOrderTypes#1770, customerOrderItemTypeCodes#1771, considerVariance#1773]
Batched: true
Location: PreparedDeltaFileIndex [hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-dl-stream-service/prp/demand-channel]
PushedFilters: [IsNotNull(name)]
ReadSchema: struct<name:string,demandStream:int,forecastCode:string,distributionChains:array<struct<internalUUID:string,salesOrganization:struct<internalRefUUID:string,id:string>,distributionChannel:struct<internalRefUUID:string,id:string>>>,customerOrderTypes:array<struct<id:string,internalRefUUID:string>>,customerOrderItemTypeCodes:array<struct<id:string,internalRefUUID:string>>,considerVariance:boolean>
(2) ColumnarToRow [codegen id : 1]
Input [7]: [name#1764, demandStream#1766, forecastCode#1767, distributionChains#1769, customerOrderTypes#1770, customerOrderItemTypeCodes#1771, considerVariance#1773]
(3) Filter [codegen id : 1]
Input [7]: [name#1764, demandStream#1766, forecastCode#1767, distributionChains#1769, customerOrderTypes#1770, customerOrderItemTypeCodes#1771, considerVariance#1773]
Condition : isnotnull(name#1764)
(4) Project
Output [7]: [name#1764, demandStream#1766, forecastCode#1767, transform(distributionChains#1769, lambdafunction(named_struct(internalUUID, lambda chain#1785.internalUUID, salesOrganizationUUID, lambda chain#1785.salesOrganization.internalRefUUID, distributionChannelUUID, lambda chain#1785.distributionChannel.internalRefUUID), lambda chain#1785, false)) AS distributionChains#1784, transform(customerOrderTypes#1770, lambdafunction(named_struct(internalUUID, lambda documentType#1798.internalRefUUID), lambda documentType#1798, false)) AS customerOrderTypes#1797, transform(customerOrderItemTypeCodes#1771, lambdafunction(named_struct(customerOrderItemTypeCodeUUID, lambda itemType#1810.internalRefUUID), lambda itemType#1810, false)) AS customerOrderItemTypeCodes#1809, considerVariance#1773]
Input [7]: [name#1764, demandStream#1766, forecastCode#1767, distributionChains#1769, customerOrderTypes#1770, customerOrderItemTypeCodes#1771, considerVariance#1773]
(5) BroadcastExchange
Input [7]: [name#1764, demandStream#1766, forecastCode#1767, distributionChains#1784, customerOrderTypes#1797, customerOrderItemTypeCodes#1809, considerVariance#1773]
Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=1746]
(6) BroadcastQueryStage
Output [7]: [name#1764, demandStream#1766, forecastCode#1767, distributionChains#1784, customerOrderTypes#1797, customerOrderItemTypeCodes#1809, considerVariance#1773]
Arguments: 0
(7) Scan ExistingRDD [codegen id : 2]
Output [2]: [demandProfile#1739, demandChannels#1740]
Arguments: [demandProfile#1739, demandChannels#1740], MapPartitionsRDD[142] at createDataFrame at DemandProfileDataReaderImpl.java:60, ExistingRDD, UnknownPartitioning(0)
(8) Filter [codegen id : 2]
Input [2]: [demandProfile#1739, demandChannels#1740]
Condition : (isnotnull(demandProfile#1739) AND ((size(demandChannels#1740, true) > 0) AND isnotnull(demandChannels#1740)))
(9) Exchange
Input [2]: [demandProfile#1739, demandChannels#1740]
Arguments: hashpartitioning(demandProfile#1739, 37), ENSURE_REQUIREMENTS, [plan_id=1757]
(10) ShuffleQueryStage
Output [2]: [demandProfile#1739, demandChannels#1740]
Arguments: 1
(11) AQEShuffleRead
Input [2]: [demandProfile#1739, demandChannels#1740]
Arguments: local
(12) BroadcastExchange
Input [2]: [demandProfile#1739, demandChannels#1740]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false]),false), [plan_id=1886]
(13) BroadcastQueryStage
Output [2]: [demandProfile#1739, demandChannels#1740]
Arguments: 3
(14) Scan parquet
Output [2]: [product#1577, plant#1578]
Batched: true
Location: InMemoryFileIndex [hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-order-qty-opt-service/out/product-plant-list/10000000091/shardId=0_1_10000000091]
ReadSchema: struct<product:string,plant:string>
(15) ColumnarToRow [codegen id : 4]
Input [2]: [product#1577, plant#1578]
(16) Scan parquet
Output [3]: [product#1585, plant#1586, demandprofile#1588]
Batched: true
Location: InMemoryFileIndex [hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-param-determination-service/out/10000000091/0_1_10000000091/demandprofile]
PushedFilters: [IsNotNull(demandprofile)]
ReadSchema: struct<product:string,plant:string,demandprofile:string>
(17) ColumnarToRow [codegen id : 3]
Input [3]: [product#1585, plant#1586, demandprofile#1588]
(18) Filter [codegen id : 3]
Input [3]: [product#1585, plant#1586, demandprofile#1588]
Condition : isnotnull(demandProfile#1588)
(19) BroadcastExchange
Input [3]: [product#1585, plant#1586, demandprofile#1588]
Arguments: HashedRelationBroadcastMode(List(coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[0, string, true], ), isnull(input[0, string, true])),false), [plan_id=1770]
(20) BroadcastQueryStage
Output [3]: [product#1585, plant#1586, demandprofile#1588]
Arguments: 2
(21) BroadcastHashJoin [codegen id : 4]
Left keys [4]: [coalesce(plant#1578, ), isnull(plant#1578), coalesce(product#1577, ), isnull(product#1577)]
Right keys [4]: [coalesce(plant#1586, ), isnull(plant#1586), coalesce(product#1585, ), isnull(product#1585)]
Join type: Inner
Join condition: None
(22) Project [codegen id : 4]
Output [1]: [demandProfile#1588]
Input [5]: [product#1577, plant#1578, product#1585, plant#1586, demandprofile#1588]
(23) HashAggregate [codegen id : 4]
Input [1]: [demandProfile#1588]
Keys [1]: [demandProfile#1588]
Functions: []
Aggregate Attributes: []
Results [1]: [demandProfile#1588]
(24) Exchange
Input [1]: [demandProfile#1588]
Arguments: hashpartitioning(demandProfile#1588, 37), ENSURE_REQUIREMENTS, [plan_id=2046]
(25) ShuffleQueryStage
Output [1]: [demandProfile#1588]
Arguments: 4
(26) AQEShuffleRead
Input [1]: [demandProfile#1588]
Arguments: coalesced
(27) HashAggregate
Input [1]: [demandProfile#1588]
Keys [1]: [demandProfile#1588]
Functions: []
Aggregate Attributes: []
Results [1]: [demandProfile#1588]
(28) BroadcastHashJoin
Left keys [1]: [demandProfile#1739]
Right keys [1]: [demandProfile#1588]
Join type: Inner
Join condition: None
(29) Project
Output [1]: [demandChannels#1740]
Input [3]: [demandProfile#1739, demandChannels#1740, demandProfile#1588]
(30) Generate
Input [1]: [demandChannels#1740]
Arguments: explode(demandChannels#1740), false, [explodedDemandChannels#1752]
(31) Filter
Input [1]: [explodedDemandChannels#1752]
Condition : isnotnull(explodedDemandChannels#1752.name)
(32) Project
Output [1]: [explodedDemandChannels#1752.name AS name#1758]
Input [1]: [explodedDemandChannels#1752]
(33) BroadcastHashJoin [codegen id : 5]
Left keys [1]: [name#1764]
Right keys [1]: [name#1758]
Join type: Inner
Join condition: None
(34) Project [codegen id : 5]
Output [7]: [name#1764, demandStream#1766, forecastCode#1767, distributionChains#1784, customerOrderTypes#1797, customerOrderItemTypeCodes#1809, considerVariance#1773]
Input [8]: [name#1764, demandStream#1766, forecastCode#1767, distributionChains#1784, customerOrderTypes#1797, customerOrderItemTypeCodes#1809, considerVariance#1773, name#1758]
(35) HashAggregate [codegen id : 5]
Input [7]: [name#1764, demandStream#1766, forecastCode#1767, distributionChains#1784, customerOrderTypes#1797, customerOrderItemTypeCodes#1809, considerVariance#1773]
Keys [7]: [name#1764, considerVariance#1773, customerOrderItemTypeCodes#1809, customerOrderTypes#1797, distributionChains#1784, forecastCode#1767, demandStream#1766]
Functions: []
Aggregate Attributes: []
Results [7]: [name#1764, considerVariance#1773, customerOrderItemTypeCodes#1809, customerOrderTypes#1797, distributionChains#1784, forecastCode#1767, demandStream#1766]
(36) Exchange
Input [7]: [name#1764, considerVariance#1773, customerOrderItemTypeCodes#1809, customerOrderTypes#1797, distributionChains#1784, forecastCode#1767, demandStream#1766]
Arguments: hashpartitioning(name#1764, considerVariance#1773, customerOrderItemTypeCodes#1809, customerOrderTypes#1797, distributionChains#1784, forecastCode#1767, demandStream#1766, 37), ENSURE_REQUIREMENTS, [plan_id=2181]
(37) ShuffleQueryStage
Output [7]: [name#1764, considerVariance#1773, customerOrderItemTypeCodes#1809, customerOrderTypes#1797, distributionChains#1784, forecastCode#1767, demandStream#1766]
Arguments: 5
(38) AQEShuffleRead
Input [7]: [name#1764, considerVariance#1773, customerOrderItemTypeCodes#1809, customerOrderTypes#1797, distributionChains#1784, forecastCode#1767, demandStream#1766]
Arguments: coalesced
(39) HashAggregate [codegen id : 6]
Input [7]: [name#1764, considerVariance#1773, customerOrderItemTypeCodes#1809, customerOrderTypes#1797, distributionChains#1784, forecastCode#1767, demandStream#1766]
Keys [7]: [name#1764, considerVariance#1773, customerOrderItemTypeCodes#1809, customerOrderTypes#1797, distributionChains#1784, forecastCode#1767, demandStream#1766]
Functions: []
Aggregate Attributes: []
Results: []
(40) Filter
Input [7]: [name#1764, demandStream#1766, forecastCode#1767, distributionChains#1769, customerOrderTypes#1770, customerOrderItemTypeCodes#1771, considerVariance#1773]
Condition : isnotnull(name#1764)
(41) Project
Output [7]: [name#1764, demandStream#1766, forecastCode#1767, transform(distributionChains#1769, lambdafunction(named_struct(internalUUID, lambda chain#1785.internalUUID, salesOrganizationUUID, lambda chain#1785.salesOrganization.internalRefUUID, distributionChannelUUID, lambda chain#1785.distributionChannel.internalRefUUID), lambda chain#1785, false)) AS distributionChains#1784, transform(customerOrderTypes#1770, lambdafunction(named_struct(internalUUID, lambda documentType#1798.internalRefUUID), lambda documentType#1798, false)) AS customerOrderTypes#1797, transform(customerOrderItemTypeCodes#1771, lambdafunction(named_struct(customerOrderItemTypeCodeUUID, lambda itemType#1810.internalRefUUID), lambda itemType#1810, false)) AS customerOrderItemTypeCodes#1809, considerVariance#1773]
Input [7]: [name#1764, demandStream#1766, forecastCode#1767, distributionChains#1769, customerOrderTypes#1770, customerOrderItemTypeCodes#1771, considerVariance#1773]
(42) BroadcastExchange
Input [7]: [name#1764, demandStream#1766, forecastCode#1767, distributionChains#1784, customerOrderTypes#1797, customerOrderItemTypeCodes#1809, considerVariance#1773]
Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=1682]
(43) Exchange
Input [2]: [demandProfile#1739, demandChannels#1740]
Arguments: hashpartitioning(demandProfile#1739, 37), ENSURE_REQUIREMENTS, [plan_id=1673]
(44) Sort
Input [2]: [demandProfile#1739, demandChannels#1740]
Arguments: [demandProfile#1739 ASC NULLS FIRST], false, 0
(45) Scan parquet
Output [2]: [product#1577, plant#1578]
Batched: true
Location: InMemoryFileIndex [hdlfs://2e93940d-4be8-4f12-830d-f0b8d392c03a.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-order-qty-opt-service/out/product-plant-list/10000000091/shardId=0_1_10000000091]
ReadSchema: struct<product:string,plant:string>
(46) Filter
Input [3]: [product#1585, plant#1586, demandprofile#1588]
Condition : isnotnull(demandProfile#1588)
(47) BroadcastExchange
Input [3]: [product#1585, plant#1586, demandprofile#1588]
Arguments: HashedRelationBroadcastMode(List(coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[0, string, true], ), isnull(input[0, string, true])),false), [plan_id=1664]
(48) BroadcastHashJoin
Left keys [4]: [coalesce(plant#1578, ), isnull(plant#1578), coalesce(product#1577, ), isnull(product#1577)]
Right keys [4]: [coalesce(plant#1586, ), isnull(plant#1586), coalesce(product#1585, ), isnull(product#1585)]
Join type: Inner
Join condition: None
(49) Project
Output [1]: [demandProfile#1588]
Input [5]: [product#1577, plant#1578, product#1585, plant#1586, demandprofile#1588]
(50) HashAggregate
Input [1]: [demandProfile#1588]
Keys [1]: [demandProfile#1588]
Functions: []
Aggregate Attributes: []
Results [1]: [demandProfile#1588]
(51) Exchange
Input [1]: [demandProfile#1588]
Arguments: hashpartitioning(demandProfile#1588, 37), ENSURE_REQUIREMENTS, [plan_id=1669]
(52) HashAggregate
Input [1]: [demandProfile#1588]
Keys [1]: [demandProfile#1588]
Functions: []
Aggregate Attributes: []
Results [1]: [demandProfile#1588]
(53) Sort
Input [1]: [demandProfile#1588]
Arguments: [demandProfile#1588 ASC NULLS FIRST], false, 0
(54) SortMergeJoin
Left keys [1]: [demandProfile#1739]
Right keys [1]: [demandProfile#1588]
Join type: Inner
Join condition: None
(55) Project
Output [1]: [demandChannels#1740]
Input [3]: [demandProfile#1739, demandChannels#1740, demandProfile#1588]
(56) Generate
Input [1]: [demandChannels#1740]
Arguments: explode(demandChannels#1740), false, [explodedDemandChannels#1752]
(57) Filter
Input [1]: [explodedDemandChannels#1752]
Condition : isnotnull(explodedDemandChannels#1752.name)
(58) Project
Output [1]: [explodedDemandChannels#1752.name AS name#1758]
Input [1]: [explodedDemandChannels#1752]
(59) BroadcastHashJoin
Left keys [1]: [name#1764]
Right keys [1]: [name#1758]
Join type: Inner
Join condition: None
(60) Project
Output [7]: [name#1764, demandStream#1766, forecastCode#1767, distributionChains#1784, customerOrderTypes#1797, customerOrderItemTypeCodes#1809, considerVariance#1773]
Input [8]: [name#1764, demandStream#1766, forecastCode#1767, distributionChains#1784, customerOrderTypes#1797, customerOrderItemTypeCodes#1809, considerVariance#1773, name#1758]
(61) HashAggregate
Input [7]: [name#1764, demandStream#1766, forecastCode#1767, distributionChains#1784, customerOrderTypes#1797, customerOrderItemTypeCodes#1809, considerVariance#1773]
Keys [7]: [name#1764, considerVariance#1773, customerOrderItemTypeCodes#1809, customerOrderTypes#1797, distributionChains#1784, forecastCode#1767, demandStream#1766]
Functions: []
Aggregate Attributes: []
Results [7]: [name#1764, considerVariance#1773, customerOrderItemTypeCodes#1809, customerOrderTypes#1797, distributionChains#1784, forecastCode#1767, demandStream#1766]
(62) Exchange
Input [7]: [name#1764, considerVariance#1773, customerOrderItemTypeCodes#1809, customerOrderTypes#1797, distributionChains#1784, forecastCode#1767, demandStream#1766]
Arguments: hashpartitioning(name#1764, considerVariance#1773, customerOrderItemTypeCodes#1809, customerOrderTypes#1797, distributionChains#1784, forecastCode#1767, demandStream#1766, 37), ENSURE_REQUIREMENTS, [plan_id=1687]
(63) HashAggregate
Input [7]: [name#1764, considerVariance#1773, customerOrderItemTypeCodes#1809, customerOrderTypes#1797, distributionChains#1784, forecastCode#1767, demandStream#1766]
Keys [7]: [name#1764, considerVariance#1773, customerOrderItemTypeCodes#1809, customerOrderTypes#1797, distributionChains#1784, forecastCode#1767, demandStream#1766]
Functions: []
Aggregate Attributes: []
Results: []
(64) AdaptiveSparkPlan
Output: []
Arguments: isFinalPlan=true