Skip to content

Moved the parquet dataset away from the hive managed folder #278

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Moved the parquet dataset away from the hive managed folder #278

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
76b12fa
Moved the dataset to a different location. Since the tables are anywa…
rchallapalli Apr 20, 2017
af51f08
Updated the README file to indicate the location of the hive generate…
rchallapalli Apr 21, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions framework/resources/Advanced/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,12 @@ For the [TPC-DS SF100 tests](tpcds/tpcds_sf100), download dataset from [Amazon S

Extract this compressed file and copy over files to MapR-FS / HDFS into `/drill/testdata/tpcds_sf100/parquet`

### TPC-DS SF 100 HIVE GENERATED PARQUET

Download the data set from [Amazon S3](http://drill-public.s3.amazonaws.com/tpcds/sf100/parquet/tpcds100_parquet_hivegen.tgz)

Extract this compressed file and copy over files to MapR-FS / HDFS into `/drill/testdata`

### Mondrian

For the [Mondrian tests](mondrian), download dataset from [Amazon S3](http://drill-public.s3.amazonaws.com/mondrian/mondrian.tgz)
Expand Down
72 changes: 48 additions & 24 deletions framework/resources/Datasources/hive_storage/tpcds100/tpcds100_hive.ddl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -624,7 +624,8 @@ create external table tpcds100_parquet.customer
c_email_address string,
c_last_review_date string
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/customer';

drop table if exists tpcds100_parquet.customer_address;
create external table tpcds100_parquet.customer_address
Expand All @@ -643,7 +644,8 @@ create external table tpcds100_parquet.customer_address
ca_gmt_offset int,
ca_location_type string
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/customer_address';

drop table if exists tpcds100_parquet.customer_demographics;
create external table tpcds100_parquet.customer_demographics
Expand All @@ -658,7 +660,8 @@ create external table tpcds100_parquet.customer_demographics
cd_dep_employed_count int,
cd_dep_college_count int
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/customer_demographics';

drop table if exists tpcds100_parquet.household_demographics;
create external table tpcds100_parquet.household_demographics
Expand All @@ -669,7 +672,8 @@ create external table tpcds100_parquet.household_demographics
hd_dep_count int,
hd_vehicle_count int
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/household_demographics';

drop table if exists tpcds100_parquet.item;
create external table tpcds100_parquet.item
Expand Down Expand Up @@ -697,7 +701,8 @@ create external table tpcds100_parquet.item
i_manager_id int,
i_product_name string
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/item';

drop table if exists tpcds100_parquet.promotion;
create external table tpcds100_parquet.promotion
Expand All @@ -722,7 +727,8 @@ create external table tpcds100_parquet.promotion
p_purpose string,
p_discount_active string
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/promotion';

drop table if exists tpcds100_parquet.time_dim;
create external table tpcds100_parquet.time_dim (
Expand All @@ -737,7 +743,8 @@ create external table tpcds100_parquet.time_dim (
t_sub_shift string,
t_meal_time string
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/time_dim';

drop table if exists tpcds100_parquet.date_dim;
create external table tpcds100_parquet.date_dim (
Expand Down Expand Up @@ -770,7 +777,8 @@ create external table tpcds100_parquet.date_dim (
d_current_quarter string,
d_current_year string
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/date_dim';

drop table if exists tpcds100_parquet.store;
create external table tpcds100_parquet.store (
Expand Down Expand Up @@ -804,7 +812,8 @@ create external table tpcds100_parquet.store (
s_gmt_offset double,
s_tax_precentage double
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/store';

drop table if exists tpcds100_parquet.store_sales;
create external table tpcds100_parquet.store_sales (
Expand Down Expand Up @@ -832,7 +841,8 @@ create external table tpcds100_parquet.store_sales (
ss_net_paid_inc_tax double,
ss_net_profit double
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/store_sales';

drop table if exists tpcds100_parquet.warehouse;
create external table tpcds100_parquet.warehouse (
Expand All @@ -851,7 +861,8 @@ create external table tpcds100_parquet.warehouse (
w_country string,
w_gmt_offset double
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/warehouse';

drop table if exists tpcds100_parquet.ship_mode;
create external table tpcds100_parquet.ship_mode (
Expand All @@ -862,23 +873,26 @@ create external table tpcds100_parquet.ship_mode (
sm_carrier string,
sm_contract string
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/ship_mode';

drop table if exists tpcds100_parquet.reason;
create external table tpcds100_parquet.reason (
r_reason_sk int,
r_reason_id string,
r_reason_desc string
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/reason';

drop table if exists tpcds100_parquet.income_band;
create external table tpcds100_parquet.income_band (
ib_income_band_sk int,
ib_lower_bound int,
ib_upper_bound int
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/income_band';

drop table if exists tpcds100_parquet.call_center;
create external table tpcds100_parquet.call_center (
Expand Down Expand Up @@ -914,7 +928,8 @@ create external table tpcds100_parquet.call_center (
cc_gmt_offset double,
cc_tax_percentage double
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/call_center';

drop table if exists tpcds100_parquet.web_site;
create external table tpcds100_parquet.web_site (
Expand Down Expand Up @@ -945,7 +960,8 @@ create external table tpcds100_parquet.web_site (
web_gmt_offset double,
web_tax_percentage double
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/web_site';

drop table if exists tpcds100_parquet.store_returns;
create external table tpcds100_parquet.store_returns (
Expand All @@ -970,7 +986,8 @@ create external table tpcds100_parquet.store_returns (
sr_store_credit double,
sr_net_loss double
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/store_returns';

drop table if exists tpcds100_parquet.web_page;
create external table tpcds100_parquet.web_page (
Expand All @@ -989,7 +1006,8 @@ create external table tpcds100_parquet.web_page (
wp_image_count int,
wp_max_ad_count int
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/web_page';

drop table if exists tpcds100_parquet.catalog_page;
create external table tpcds100_parquet.catalog_page (
Expand All @@ -1003,7 +1021,8 @@ create external table tpcds100_parquet.catalog_page (
cp_description string,
cp_type string
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/catalog_page';

drop table if exists tpcds100_parquet.inventory;
create external table tpcds100_parquet.inventory (
Expand All @@ -1012,7 +1031,8 @@ create external table tpcds100_parquet.inventory (
inv_warehouse_sk int,
inv_quantity_on_hand int
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/inventory';

drop table if exists tpcds100_parquet.catalog_returns;
create external table tpcds100_parquet.catalog_returns (
Expand Down Expand Up @@ -1044,7 +1064,8 @@ create external table tpcds100_parquet.catalog_returns (
cr_store_credit double,
cr_net_loss double
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/catalog_returns';

drop table if exists tpcds100_parquet.web_returns;
create external table tpcds100_parquet.web_returns (
Expand Down Expand Up @@ -1073,7 +1094,8 @@ create external table tpcds100_parquet.web_returns (
wr_account_credit double,
wr_net_loss double
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/web_returns';

drop table if exists tpcds100_parquet.web_sales;
create external table tpcds100_parquet.web_sales (
Expand Down Expand Up @@ -1112,7 +1134,8 @@ create external table tpcds100_parquet.web_sales (
ws_net_paid_inc_ship_tax double,
ws_net_profit double
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/web_sales';

drop table if exists tpcds100_parquet.catalog_sales;
create external table tpcds100_parquet.catalog_sales (
Expand Down Expand Up @@ -1151,4 +1174,5 @@ create external table tpcds100_parquet.catalog_sales (
cs_net_paid_inc_ship_tax double,
cs_net_profit double
)
STORED AS PARQUET;
STORED AS PARQUET
LOCATION '/drill/testdata/tpcds100_parquet_hivegen/catalog_sales';