Revert "MD-2954: Fixed TPCDS100 hive datagen script (#337)" (#402)

This reverts commit 7c5ea70.

Author: prasadns14

framework/resources/Datasources/hive_storage/change_metadata2.ddl

@@ -34,45 +34,45 @@ ALTER TABLE remove_columns2 REPLACE COLUMNS (
 ALTER TABLE modify_columntypes1 REPLACE COLUMNS (
   int_col string,
   bigint_col varchar(50),
-  date_col date,
-  time_col string,
+  date_col int,
+  time_col bigint,
   timestamp_col varchar(20),
-  interval_col string,
-  varchar_col string,
-  float_col float,
-  double_col double,
-  bool_col boolean
+  interval_col date,
+  varchar_col int,
+  float_col int,
+  double_col float,
+  bool_col int
 );
 
 ALTER TABLE modify_columntypes2 REPLACE COLUMNS (
   int_col float,
-  bigint_col bigint,
+  bigint_col int,
   date_col varchar(15),
-  time_col string,
-  timestamp_col timestamp,
-  interval_col string,
+  time_col timestamp,
+  timestamp_col date,
+  interval_col boolean,
   varchar_col varchar(5),
   float_col string,
-  double_col double,
-  bool_col boolean
+  double_col bigint,
+  bool_col float
 );
 
 alter table seq_modify_columntypes1 replace columns (
     l_orderkey float,
     l_partkey double,
     l_suppkey varchar(1),
     l_linenumber string,
-    l_quantity double,
-    l_extendedprice double,
-    l_discount double,
+    l_quantity bigint,
+    l_extendedprice int,
+    l_discount float,
     l_tax varchar(20),
-    l_returnflag string,
-    l_linestatus string,
-    l_shipdate date,
+    l_returnflag boolean,
+    l_linestatus int,
+    l_shipdate timestamp,
     l_commitdate string,
-    l_receiptdate date,
+    l_receiptdate int,
     l_shipinstruct varchar(2),
-    l_shipmode string,
+    l_shipmode boolean,
     l_comment double
 );
 ALTER TABLE dpp_compressed_mktevents ADD COLUMNS (newcol STRING);
@@ -87,10 +87,7 @@ ALTER TABLE changemetadata_orders_partitioned ADD PARTITION (year=1997) location
 
 ALTER TABLE changemetadata_orders_partitioned1 replace columns (
     o_orderkey INT,
-    o_custkey INT
-);
-
-ALTER TABLE changemetadata_orders_partitioned1 add columns (
+    o_custkey INT,
     o_orderdate STRING,
     o_orderpriority STRING,
     o_clerk STRING,
@@ -106,9 +103,9 @@ ALTER TABLE changemetadata_orders_partitioned2 replace columns (
     o_custkey STRING,
     o_orderstatus STRING,
     o_totalprice DOUBLE,
-    o_orderdate STRING,
+    o_orderdate BOOLEAN,
     o_orderpriority VARCHAR(1),
     o_clerk DOUBLE,
     o_shippriority INT
 );
-ALTER TABLE changemetadata_orders_partitioned2 DROP IF EXISTS PARTITION(year=1990);
+ALTER TABLE changemetadata_orders_partitioned2 DROP IF EXISTS PARTITION(year=1990)

framework/resources/Datasources/hive_storage/tpcds100/tpcds100_hive.ddl

@@ -624,9 +624,7 @@ create external table tpcds100_parquet.customer
     c_email_address string,
     c_last_review_date string
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/customer'
-TBLPROPERTIES ("parquet.page.size"="4096","parquet.block.size"="12288");
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.customer_address;
 create external table tpcds100_parquet.customer_address 
@@ -645,9 +643,7 @@ create external table tpcds100_parquet.customer_address
     ca_gmt_offset int,
     ca_location_type string
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/customer_address'
-TBLPROPERTIES ("parquet.dictionary.page.size"="65536");
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.customer_demographics;
 create external table tpcds100_parquet.customer_demographics 
@@ -662,9 +658,7 @@ create external table tpcds100_parquet.customer_demographics
     cd_dep_employed_count int,
     cd_dep_college_count int
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/customer_demographics'
-TBLPROPERTIES ("parquet.enable.dictionary"="false", "parquet.compression"="SNAPPY", "parquet.page.size"="4096","parquet.block.size"="12288");
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.household_demographics;
 create external table tpcds100_parquet.household_demographics 
@@ -675,17 +669,15 @@ create external table tpcds100_parquet.household_demographics
     hd_dep_count int,
     hd_vehicle_count int
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/household_demographics'
-TBLPROPERTIES ("parquet.compression"="GZIP");
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.item;
-create external table tpcds100_parquet.item
+create external table tpcds100_parquet.item 
 (
     i_item_sk int,
     i_item_id string,
-    i_rec_start_date date,
-    i_rec_end_date date,
+    i_rec_start_date string,
+    i_rec_end_date string,
     i_item_desc string,
     i_current_price double,
     i_wholesale_cost double,
@@ -705,8 +697,7 @@ create external table tpcds100_parquet.item
     i_manager_id int,
     i_product_name string
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/item';
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.promotion;
 create external table tpcds100_parquet.promotion 
@@ -731,8 +722,7 @@ create external table tpcds100_parquet.promotion
     p_purpose string,
     p_discount_active string
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/promotion';
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.time_dim;
 create external table tpcds100_parquet.time_dim (
@@ -747,9 +737,7 @@ create external table tpcds100_parquet.time_dim (
     t_sub_shift string, 
     t_meal_time string
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/time_dim'
-TBLPROPERTIES ("parquet.enable.dictionary"="false", "parquet.compression"="GZIP", "parquet.block.size"="32768", "parquet.page.size"="1024");
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.date_dim;
 create external table tpcds100_parquet.date_dim (
@@ -782,12 +770,10 @@ create external table tpcds100_parquet.date_dim (
     d_current_quarter string, 
     d_current_year string
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/date_dim'
-TBLPROPERTIES ("parquet.enable.dictionary"="false", "parquet.page.size"="134217728");
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.store;
-create external table tpcds100_parquet.store (
+create external table tpcds100_parquet.store ( 
     s_store_sk int, 
     s_store_id string, 
     s_rec_start_date string, 
@@ -818,12 +804,10 @@ create external table tpcds100_parquet.store (
     s_gmt_offset double, 
     s_tax_precentage double
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/store'
-TBLPROPERTIES ("parquet.dictionary.page.size"="32768", "parquet.block.size"="32768", "parquet.page.size"="32768");
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.store_sales;
-create external table tpcds100_parquet.store_sales (
+create external table tpcds100_parquet.store_sales ( 
     ss_sold_date_sk int, 
     ss_sold_time_sk int, 
     ss_item_sk int, 
@@ -841,18 +825,17 @@ create external table tpcds100_parquet.store_sales (
     ss_ext_discount_amt double, 
     ss_ext_sales_price double, 
     ss_ext_wholesale_cost double, 
-    ss_ext_list_price double,
+    ss_ext_list_price double, 
     ss_ext_tax double, 
     ss_coupon_amt double, 
     ss_net_paid double, 
     ss_net_paid_inc_tax double, 
     ss_net_profit double
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/store_sales';
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.warehouse;
-create external table tpcds100_parquet.warehouse (
+create external table tpcds100_parquet.warehouse ( 
     w_warehouse_sk int,
     w_warehouse_id string,
     w_warehouse_name string,
@@ -868,8 +851,7 @@ create external table tpcds100_parquet.warehouse (
     w_country string,
     w_gmt_offset double       
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/warehouse';
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.ship_mode;
 create external table tpcds100_parquet.ship_mode (
@@ -880,29 +862,26 @@ create external table tpcds100_parquet.ship_mode (
     sm_carrier string,
     sm_contract string     
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/ship_mode';
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.reason;
 create external table tpcds100_parquet.reason (
     r_reason_sk int,
     r_reason_id string,
     r_reason_desc string  
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/reason';
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.income_band;
-create external table tpcds100_parquet.income_band (
+create external table tpcds100_parquet.income_band ( 
     ib_income_band_sk int,
     ib_lower_bound int,
     ib_upper_bound int         
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/income_band';
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.call_center;
-create external table tpcds100_parquet.call_center (
+create external table tpcds100_parquet.call_center ( 
     cc_call_center_sk int,
     cc_call_center_id string,
     cc_rec_start_date string,
@@ -935,11 +914,10 @@ create external table tpcds100_parquet.call_center (
     cc_gmt_offset double,
     cc_tax_percentage double 
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/call_center';
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.web_site;
-create external table tpcds100_parquet.web_site (
+create external table tpcds100_parquet.web_site (  
     web_site_sk int,
     web_site_id string,
     web_rec_start_date string,
@@ -967,11 +945,10 @@ create external table tpcds100_parquet.web_site (
     web_gmt_offset double,
     web_tax_percentage double
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/web_site';
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.store_returns;
-create external table tpcds100_parquet.store_returns (
+create external table tpcds100_parquet.store_returns ( 
     sr_returned_date_sk int,
     sr_return_time_sk int,
     sr_item_sk int,
@@ -993,11 +970,10 @@ create external table tpcds100_parquet.store_returns (
     sr_store_credit double,
     sr_net_loss double    
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/store_returns';
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.web_page;
-create external table tpcds100_parquet.web_page (
+create external table tpcds100_parquet.web_page ( 
     wp_web_page_sk int,
     wp_web_page_id string,
     wp_rec_start_date string,
@@ -1013,8 +989,7 @@ create external table tpcds100_parquet.web_page (
     wp_image_count int,
     wp_max_ad_count int         
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/web_page';
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.catalog_page;
 create external table tpcds100_parquet.catalog_page (
@@ -1028,23 +1003,19 @@ create external table tpcds100_parquet.catalog_page (
     cp_description string,
     cp_type string        
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/catalog_page'
-TBLPROPERTIES ("parquet.dictionary.page.size"="5242880", "parquet.page.size"="262144", "parquet.block.size"="262144");
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.inventory;
-create external table tpcds100_parquet.inventory (
+create external table tpcds100_parquet.inventory ( 
     inv_date_sk int,
     inv_item_sk int,
     inv_warehouse_sk int,
     inv_quantity_on_hand int            
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/inventory'
-TBLPROPERTIES ("parquet.block.size"="10485760");
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.catalog_returns;
-create external table tpcds100_parquet.catalog_returns (
+create external table tpcds100_parquet.catalog_returns ( 
     cr_returned_date_sk int,
     cr_returned_time_sk int,
     cr_item_sk int,
@@ -1073,11 +1044,10 @@ create external table tpcds100_parquet.catalog_returns (
     cr_store_credit double,
     cr_net_loss double       
 )
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/catalog_returns';
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.web_returns;
-create table tpcds100_parquet.web_returns (
+create external table tpcds100_parquet.web_returns ( 
      wr_returned_date_sk int,
      wr_returned_time_sk int,
      wr_item_sk int,
@@ -1103,8 +1073,7 @@ create table tpcds100_parquet.web_returns (
      wr_account_credit double,
      wr_net_loss double 
 )       
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/web_returns';
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.web_sales;
 create external table tpcds100_parquet.web_sales (
@@ -1143,11 +1112,10 @@ create external table tpcds100_parquet.web_sales (
     ws_net_paid_inc_ship_tax double,
     ws_net_profit double       
 )       
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/web_sales';
+STORED AS PARQUET;
 
 drop table if exists tpcds100_parquet.catalog_sales;
-create external table tpcds100_parquet.catalog_sales (
+create external table tpcds100_parquet.catalog_sales ( 
     cs_sold_date_sk int,
     cs_sold_time_sk int,
     cs_ship_date_sk int,
@@ -1183,5 +1151,4 @@ create external table tpcds100_parquet.catalog_sales (
     cs_net_paid_inc_ship_tax double,
     cs_net_profit double        
 )       
-STORED AS PARQUET
-LOCATION '/drill/testdata/tpcds_sf100/parquet/catalog_sales';
+STORED AS PARQUET;