Classification CoverType.py - # coding: utf-8 # # Problem...

# coding: utf-8
# ## Problem Statement
#
# In this programming assignment, your task is to classify
geographical locations according to their predicted tree cover
using Gradient Boosting and Random Forest classifiers. You are
expected to fill in functions that would complete this task. All
of the necessary helper code is included in this notebook.
However, we advise you to go over the slides, lecture material,
the EdX videos and the corresponding notebooks before you attempt
this Programming Assignment. You can find information about the
dataset to be used in the following links:
#
# * **Dataset:**
#
# * **Dataset description:**
-
databases/covtype/covtype.info
# ## Notebook Setup
# In[1]:
# To time the entire solution
import time
start_nb = time.time()
# In[2]:
import os
os.environ["PYSPARK_PYTHON"]="python3"
os.environ["PYSPARK_DRIVER_PYTHON"] = "python3"
from pyspark import SparkContext
sc=SparkContext()
# In[3]:

from pyspark.mllib.linalg import Vectors
from pyspark.mllib.regression import LabeledPoint
from pyspark.mllib.tree import GradientBoostedTrees,
GradientBoostedTreesModel
from pyspark.mllib.tree import RandomForest, RandomForestModel
from pyspark.mllib.util import MLUtils
# import os
import pickle
from os.path import exists
get_ipython().magic('config IPCompleter.greedy=True')
# In[4]:
#define a dictionary of cover types
CoverTypes={1.0: 'Spruce/Fir',
2.0: 'Lodgepole Pine',
3.0: 'Ponderosa Pine',
4.0: 'Cottonwood/Willow',
5.0: 'Aspen',
6.0: 'Douglas-fir',
7.0: 'Krummholz' }
print('Tree Cover Types:', CoverTypes)
# ## Collecting Data
# In[5]:
# Break up features that are made out of several binary features.
def get_columns(cols_txt):
cols=[a.strip() for a in cols_txt.split(',')]
colDict={a:[a] for a in cols}
colDict['Soil_Type (40 binary columns)'] = ['ST_'+str(i) for i
in range(40)]
colDict['Wilderness_Area (4 binarycolumns)'] = ['WA_'+str(i)
for i in range(4)]
columns=[]
for item in cols:

columns = columns + colDict[item]
return columns
#print(columns)
# In[6]:
# Define the feature names
cols_txt="""
Elevation, Aspect, Slope, Horizontal_Distance_To_Hydrology,
Vertical_Distance_To_Hydrology, Horizontal_Distance_To_Roadways,
Hillshade_9am, Hillshade_Noon, Hillshade_3pm,
Horizontal_Distance_To_Fire_Points, Wilderness_Area (4
binarycolumns),
Soil_Type (40 binary columns), Cover_Type
"""
columns = get_columns(cols_txt)
# In[7]:
# Read the file into an RDD
# When using sc.textRead you need to use an absolute path.