Basic function to preprocess tabular data before assembling it in a `DataLoaders`.
!pip install git+http://github.com/fastai/fastai2
!pip install git+http://github.com/fastai/fastcore
from nbdev.showdoc import *

Initial preprocessing

make_date[source]

make_date(df, date_field)

Make sure df[date_field] is of the right date type.

df = pd.DataFrame({'date': ['2019-12-04', '2019-11-29', '2019-11-15', '2019-10-24']})
make_date(df, 'date')
test_eq(df['date'].dtype, np.dtype('datetime64[ns]'))

add_datepart[source]

add_datepart(df, field_name, prefix=None, drop=True, time=False)

Helper function that adds columns relevant to a date in the column field_name of df.

df = pd.DataFrame({'date': ['2019-12-04', '2019-11-29', '2019-11-15', '2019-10-24']})
df = add_datepart(df, 'date')
test_eq(df.columns, ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Dayofyear', 'Is_month_end', 'Is_month_start', 
            'Is_quarter_end', 'Is_quarter_start', 'Is_year_end', 'Is_year_start', 'Elapsed'])
df.head()
Year Month Week Day Dayofweek Dayofyear Is_month_end Is_month_start Is_quarter_end Is_quarter_start Is_year_end Is_year_start Elapsed
0 2019 12 49 4 2 338 False False False False False False 1575417600
1 2019 11 48 29 4 333 False False False False False False 1574985600
2 2019 11 46 15 4 319 False False False False False False 1573776000
3 2019 10 43 24 3 297 False False False False False False 1571875200

add_elapsed_times[source]

add_elapsed_times(df, field_names, date_field, base_field)

Add in df for each event in field_names the elapsed time according to date_field grouped by base_field

df = pd.DataFrame({'date': ['2019-12-04', '2019-11-29', '2019-11-15', '2019-10-24'],
                   'event': [False, True, False, True], 'base': [1,1,2,2]})
df = add_elapsed_times(df, ['event'], 'date', 'base')
df
date event base Afterevent Beforeevent event_bw event_fw
0 2019-12-04 False 1 5 0 1.0 0.0
1 2019-11-29 True 1 0 0 1.0 1.0
2 2019-11-15 False 2 22 0 1.0 0.0
3 2019-10-24 True 2 0 0 1.0 1.0

cont_cat_split[source]

cont_cat_split(df, max_card=20, dep_var=None)

Helper function that returns column names of cont and cat variables from given df.

class Tabular[source]

Tabular(df, procs=None, cat_names=None, cont_names=None, y_names=None, y_block=None, splits=None, do_setup=True, device=None, inplace=False, reduce_memory=True) :: CollBase

A DataFrame wrapper that knows which cols are cont/cat/y, and returns rows in __getitem__

class TabularPandas[source]

TabularPandas(df, procs=None, cat_names=None, cont_names=None, y_names=None, y_block=None, splits=None, do_setup=True, device=None, inplace=False, reduce_memory=True) :: Tabular

A DataFrame wrapper that knows which cols are cont/cat/y, and returns rows in __getitem__

df = pd.DataFrame({'a':[0,1,2,0,2], 'b':[0,0,0,0,1]})
to = TabularPandas(df, cat_names='a')
t = pickle.loads(pickle.dumps(to))
test_eq(t.items,to.items)
test_eq(to.all_cols,to[['a']])

class TabularProc[source]

TabularProc(enc=None, dec=None, split_idx=None, order=None) :: InplaceTransform

Base class to write a non-lazy tabular processor for dataframes

setups[source]

setups(to:Tabular)

encodes[source]

encodes(to:Tabular)

decodes[source]

decodes(to:Tabular)

class Categorify[source]

Categorify(enc=None, dec=None, split_idx=None, order=None) :: TabularProc

Transform the categorical variables to that type.

df = pd.DataFrame({'a':[0,1,2,0,2]})
to = TabularPandas(df, Categorify, 'a')
cat = to.procs.categorify
test_eq(cat['a'], ['#na#',0,1,2])
test_eq(to['a'], [1,2,3,1,3])
to.show()
a
0 0
1 1
2 2
3 0
4 2
df1 = pd.DataFrame({'a':[1,0,3,-1,2]})
to1 = to.new(df1)
to1.process()
#Values that weren't in the training df are sent to 0 (na)
test_eq(to1['a'], [2,1,0,0,3])
to2 = cat.decode(to1)
test_eq(to2['a'], [1,0,'#na#','#na#',2])
#test with splits
cat = Categorify()
df = pd.DataFrame({'a':[0,1,2,3,2]})
to = TabularPandas(df, cat, 'a', splits=[[0,1,2],[3,4]])
test_eq(cat['a'], ['#na#',0,1,2])
test_eq(to['a'], [1,2,3,0,3])
df = pd.DataFrame({'a':pd.Categorical(['M','H','L','M'], categories=['H','M','L'], ordered=True)})
to = TabularPandas(df, Categorify, 'a')
cat = to.procs.categorify
test_eq(cat['a'], ['#na#','H','M','L'])
test_eq(to.items.a, [2,1,3,2])
to2 = cat.decode(to)
test_eq(to2['a'], ['M','H','L','M'])
#test with targets
cat = Categorify()
df = pd.DataFrame({'a':[0,1,2,3,2], 'b': ['a', 'b', 'a', 'b', 'b']})
to = TabularPandas(df, cat, 'a', splits=[[0,1,2],[3,4]], y_names='b')
test_eq(to.vocab, ['a', 'b'])
test_eq(to['b'], [0,1,0,1,1])
to2 = to.procs.decode(to)
test_eq(to2['b'], ['a', 'b', 'a', 'b', 'b'])
#test with targets and train
cat = Categorify()
df = pd.DataFrame({'a':[0,1,2,3,2], 'b': ['a', 'b', 'a', 'c', 'b']})
to = TabularPandas(df, cat, 'a', splits=[[0,1,2],[3,4]], y_names='b')
test_eq(to.vocab, ['a', 'b'])

class NormalizeTab[source]

NormalizeTab(enc=None, dec=None, split_idx=None, order=None) :: TabularProc

Normalize the continuous variables.

setups[source]

setups(to:Tabular)

encodes[source]

encodes(to:Tabular)

decodes[source]

decodes(to:Tabular)

norm = Normalize()
df = pd.DataFrame({'a':[0,1,2,3,4]})
to = TabularPandas(df, norm, cont_names='a')
x = np.array([0,1,2,3,4])
m,s = x.mean(),x.std()
test_eq(norm.means['a'], m)
test_close(norm.stds['a'], s)
test_close(to['a'].values, (x-m)/s)
df1 = pd.DataFrame({'a':[5,6,7]})
to1 = to.new(df1)
to1.process()
test_close(to1['a'].values, (np.array([5,6,7])-m)/s)
to2 = norm.decode(to1)
test_close(to2['a'].values, [5,6,7])
norm = Normalize()
df = pd.DataFrame({'a':[0,1,2,3,4]})
to = TabularPandas(df, norm, cont_names='a', splits=[[0,1,2],[3,4]])
x = np.array([0,1,2])
m,s = x.mean(),x.std()
test_eq(norm.means['a'], m)
test_close(norm.stds['a'], s)
test_close(to['a'].values, (np.array([0,1,2,3,4])-m)/s)

class FillStrategy[source]

FillStrategy()

Namespace containing the various filling strategies.

class FillMissing[source]

FillMissing(fill_strategy='median', add_col=True, fill_vals=None) :: TabularProc

Fill the missing values in continuous columns.

fill1,fill2,fill3 = (FillMissing(fill_strategy=s) 
                     for s in [FillStrategy.median, FillStrategy.constant, FillStrategy.mode])
df = pd.DataFrame({'a':[0,1,np.nan,1,2,3,4]})
df1 = df.copy(); df2 = df.copy()
tos = (TabularPandas(df, fill1, cont_names='a'),
       TabularPandas(df1, fill2, cont_names='a'),
       TabularPandas(df2, fill3, cont_names='a'))
test_eq(fill1.na_dict, {'a': 1.5})
test_eq(fill2.na_dict, {'a': 0})
test_eq(fill3.na_dict, {'a': 1.0})

for t in tos: test_eq(t.cat_names, ['a_na'])

for to_,v in zip(tos, [1.5, 0., 1.]):
    test_eq(to_['a'].values, np.array([0, 1, v, 1, 2, 3, 4]))
    test_eq(to_['a_na'].values, np.array([0, 0, 1, 0, 0, 0, 0]))
fill = FillMissing() 
df = pd.DataFrame({'a':[0,1,np.nan,1,2,3,4], 'b': [0,1,2,3,4,5,6]})
to = TabularPandas(df, fill, cont_names=['a', 'b'])
test_eq(fill.na_dict, {'a': 1.5})
test_eq(to.cat_names, ['a_na'])
test_eq(to['a'].values, np.array([0, 1, 1.5, 1, 2, 3, 4]))
test_eq(to['a_na'].values, np.array([0, 0, 1, 0, 0, 0, 0]))
test_eq(to['b'].values, np.array([0,1,2,3,4,5,6]))
procs = [Normalize, Categorify, FillMissing, noop]
df = pd.DataFrame({'a':[0,1,2,1,1,2,0], 'b':[0,1,np.nan,1,2,3,4]})
to = TabularPandas(df, procs, cat_names='a', cont_names='b')

#Test setup and apply on df_main
test_eq(to.cat_names, ['a', 'b_na'])
test_eq(to['a'], [1,2,3,2,2,3,1])
test_eq(to['b_na'], [1,1,2,1,1,1,1])
x = np.array([0,1,1.5,1,2,3,4])
m,s = x.mean(),x.std()
test_close(to['b'].values, (x-m)/s)
test_eq(to.classes, {'a': ['#na#',0,1,2], 'b_na': ['#na#',False,True]})
#Test apply on y_names
df = pd.DataFrame({'a':[0,1,2,1,1,2,0], 'b':[0,1,np.nan,1,2,3,4], 'c': ['b','a','b','a','a','b','a']})
to = TabularPandas(df, procs, 'a', 'b', y_names='c')

test_eq(to.cat_names, ['a', 'b_na'])
test_eq(to['a'], [1,2,3,2,2,3,1])
test_eq(to['b_na'], [1,1,2,1,1,1,1])
test_eq(to['c'], [1,0,1,0,0,1,0])
x = np.array([0,1,1.5,1,2,3,4])
m,s = x.mean(),x.std()
test_close(to['b'].values, (x-m)/s)
test_eq(to.classes, {'a': ['#na#',0,1,2], 'b_na': ['#na#',False,True]})
test_eq(to.vocab, ['a','b'])
df = pd.DataFrame({'a':[0,1,2,1,1,2,0], 'b':[0,1,np.nan,1,2,3,4], 'c': ['b','a','b','a','a','b','a']})
to = TabularPandas(df, procs, 'a', 'b', y_names='c')

test_eq(to.cat_names, ['a', 'b_na'])
test_eq(to['a'], [1,2,3,2,2,3,1])
test_eq(df.a.dtype,int)
test_eq(to['b_na'], [1,1,2,1,1,1,1])
test_eq(to['c'], [1,0,1,0,0,1,0])
df = pd.DataFrame({'a':[0,1,2,1,1,2,0], 'b':[0,np.nan,1,1,2,3,4], 'c': ['b','a','b','a','a','b','a']})
to = TabularPandas(df, procs, cat_names='a', cont_names='b', y_names='c', splits=[[0,1,4,6], [2,3,5]])

test_eq(to.cat_names, ['a', 'b_na'])
test_eq(to['a'], [1,2,2,1,0,2,0])
test_eq(df.a.dtype,int)
test_eq(to['b_na'], [1,2,1,1,1,1,1])
test_eq(to['c'], [1,0,0,0,1,0,1])

class ReadTabBatch[source]

ReadTabBatch(to) :: ItemTransform

A transform that always take tuples as items

from torch.utils.data.dataloader import _MultiProcessingDataLoaderIter,_SingleProcessDataLoaderIter,_DatasetKind
_loaders = (_MultiProcessingDataLoaderIter,_SingleProcessDataLoaderIter)

class TabDataLoader[source]

TabDataLoader(dataset, bs=16, shuffle=False, after_batch=None, num_workers=0, verbose=False, do_setup=True, pin_memory=False, timeout=0, batch_size=None, drop_last=False, indexed=None, n=None, device=None, wif=None, before_iter=None, after_item=None, before_batch=None, after_iter=None, create_batches=None, create_item=None, create_batch=None, retain=None, get_idxs=None, sample=None, shuffle_fn=None, do_batch=None) :: TfmdDL

Transformed DataLoader

Integration example

path = untar_data(URLs.ADULT_SAMPLE)
df = pd.read_csv(path/'adult.csv')
df_main,df_test = df.iloc[:10000].copy(),df.iloc[10000:].copy()
df_test.drop('salary', axis=1, inplace=True)
df_main.head()
age workclass fnlwgt education education-num marital-status occupation relationship race sex capital-gain capital-loss hours-per-week native-country salary
0 49 Private 101320 Assoc-acdm 12.0 Married-civ-spouse NaN Wife White Female 0 1902 40 United-States >=50k
1 44 Private 236746 Masters 14.0 Divorced Exec-managerial Not-in-family White Male 10520 0 45 United-States >=50k
2 38 Private 96185 HS-grad NaN Divorced NaN Unmarried Black Female 0 0 32 United-States <50k
3 38 Self-emp-inc 112847 Prof-school 15.0 Married-civ-spouse Prof-specialty Husband Asian-Pac-Islander Male 0 0 40 United-States >=50k
4 42 Self-emp-not-inc 82297 7th-8th NaN Married-civ-spouse Other-service Wife Black Female 0 0 50 United-States <50k
cat_names = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race']
cont_names = ['age', 'fnlwgt', 'education-num']
procs = [Categorify, FillMissing, Normalize]
splits = RandomSplitter()(range_of(df_main))
to = TabularPandas(df_main, procs, cat_names, cont_names, y_names="salary", splits=splits)
dls = to.dataloaders()
dls.valid.show_batch()
workclass education marital-status occupation relationship race education-num_na age fnlwgt education-num salary
0 Local-gov Some-college Married-civ-spouse Protective-serv Husband White False 41.000000 33658.005550 10.0 >=50k
1 Local-gov Masters Married-civ-spouse Prof-specialty Husband White False 43.000000 174394.999364 14.0 <50k
2 Private Bachelors Divorced Prof-specialty Unmarried White False 50.000000 202044.000151 13.0 <50k
3 Federal-gov 11th Never-married Sales Not-in-family Black False 50.000000 166419.000381 7.0 <50k
4 Private Some-college Married-civ-spouse #na# Husband White True 46.000000 149948.999522 10.0 <50k
5 Private Bachelors Married-civ-spouse #na# Husband White True 45.000000 235645.999841 10.0 >=50k
6 Local-gov HS-grad Married-civ-spouse Protective-serv Husband White False 31.000000 243665.000438 9.0 >=50k
7 Private Some-college Never-married Sales Own-child Black False 20.000000 211293.000061 10.0 <50k
8 Private Bachelors Married-civ-spouse Prof-specialty Husband White False 54.999999 193129.999986 13.0 >=50k
9 Local-gov Bachelors Never-married Craft-repair Not-in-family White False 31.000000 158290.998489 13.0 >=50k
to.show()
workclass education marital-status occupation relationship race education-num_na age fnlwgt education-num salary
2091 Private Assoc-voc Never-married Other-service Unmarried Black False 33.0 149220.0 11.0 <50k
8256 Private HS-grad Never-married Other-service Unmarried White False 23.0 197904.0 9.0 <50k
9997 Private Some-college Never-married Adm-clerical Unmarried White False 38.0 188503.0 10.0 <50k
4172 Private Some-college Married-civ-spouse Prof-specialty Husband White False 32.0 127451.0 10.0 >=50k
3958 Private Bachelors Married-civ-spouse Prof-specialty Wife White False 36.0 166988.0 13.0 >=50k
545 Private Some-college Never-married Machine-op-inspct Not-in-family Asian-Pac-Islander False 26.0 162302.0 10.0 <50k
2574 Private HS-grad Married-civ-spouse Exec-managerial Husband White False 37.0 170174.0 9.0 >=50k
9944 Private Some-college Married-civ-spouse Adm-clerical Wife White False 41.0 168324.0 10.0 >=50k
598 State-gov Bachelors Married-civ-spouse Prof-specialty Husband White True 29.0 303446.0 10.0 <50k
6201 Federal-gov HS-grad Married-civ-spouse Armed-Forces Other-relative Amer-Indian-Eskimo False 30.0 76313.0 9.0 <50k
row = to.items.iloc[0]
to.decode_row(row)
age                             33
workclass                  Private
fnlwgt                      149220
education                Assoc-voc
education-num                   11
marital-status       Never-married
occupation           Other-service
relationship             Unmarried
race                         Black
sex                         Female
capital-gain                     0
capital-loss                     0
hours-per-week                  40
native-country       United-States
salary                        <50k
education-num_na             False
Name: 2091, dtype: object
to_tst = to.new(df_test)
to_tst.process()
to_tst.items.head()
age workclass fnlwgt education education-num marital-status occupation relationship race sex capital-gain capital-loss hours-per-week native-country education-num_na
10000 0.464271 5 1.353993 10 1.170462 3 2 1 2 Male 0 0 40 Philippines 1
10001 -0.929853 5 1.266179 12 -0.439735 3 15 1 4 Male 0 0 40 United-States 1
10002 1.051270 5 0.153469 2 -1.244833 1 9 2 5 Female 0 0 37 United-States 1
10003 0.537646 5 -0.283575 12 -0.439735 7 2 5 5 Female 0 0 43 United-States 1
10004 0.757770 6 1.457460 9 0.365364 3 5 1 5 Male 0 0 60 United-States 1
tst_dl = dls.valid.new(to_tst)
tst_dl.show_batch()
workclass education marital-status occupation relationship race education-num_na age fnlwgt education-num
0 Private Bachelors Married-civ-spouse Adm-clerical Husband Asian-Pac-Islander False 45.000000 338105.006310 13.0
1 Private HS-grad Married-civ-spouse Transport-moving Husband Other False 26.000000 328662.999829 9.0
2 Private 11th Divorced Other-service Not-in-family White False 53.000001 209021.999284 7.0
3 Private HS-grad Widowed Adm-clerical Unmarried White False 46.000000 162029.999909 9.0
4 Self-emp-inc Assoc-voc Married-civ-spouse Exec-managerial Husband White False 49.000000 349229.995866 11.0
5 Local-gov Some-college Married-civ-spouse Exec-managerial Husband White False 34.000000 124827.001629 10.0
6 Self-emp-inc Some-college Married-civ-spouse Sales Husband White False 53.000001 290639.998802 10.0
7 Private Some-college Never-married Sales Own-child White False 19.000000 106272.998291 10.0
8 Private Some-college Married-civ-spouse Protective-serv Husband Black False 71.999999 53683.994218 10.0
9 Private Some-college Never-married Sales Own-child White False 20.000000 505979.998058 10.0

Other target types

Multi-label categories

one-hot encoded label

def _mock_multi_label(df):
    sal,sex,white = [],[],[]
    for row in df.itertuples():
        sal.append(row.salary == '>=50k')
        sex.append(row.sex == ' Male')
        white.append(row.race == ' White')
    df['salary'] = np.array(sal)
    df['male']   = np.array(sex)
    df['white']  = np.array(white)
    return df
path = untar_data(URLs.ADULT_SAMPLE)
df = pd.read_csv(path/'adult.csv')
df_main,df_test = df.iloc[:10000].copy(),df.iloc[10000:].copy()
df_main = _mock_multi_label(df_main)
df_main.head()
age workclass fnlwgt education education-num marital-status occupation relationship race sex capital-gain capital-loss hours-per-week native-country salary male white
0 49 Private 101320 Assoc-acdm 12.0 Married-civ-spouse NaN Wife White Female 0 1902 40 United-States True False True
1 44 Private 236746 Masters 14.0 Divorced Exec-managerial Not-in-family White Male 10520 0 45 United-States True True True
2 38 Private 96185 HS-grad NaN Divorced NaN Unmarried Black Female 0 0 32 United-States False False False
3 38 Self-emp-inc 112847 Prof-school 15.0 Married-civ-spouse Prof-specialty Husband Asian-Pac-Islander Male 0 0 40 United-States True True False
4 42 Self-emp-not-inc 82297 7th-8th NaN Married-civ-spouse Other-service Wife Black Female 0 0 50 United-States False False False

encodes[source]

encodes(to:Tabular)

decodes[source]

decodes(to:Tabular)

cat_names = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race']
cont_names = ['age', 'fnlwgt', 'education-num']
procs = [Categorify, FillMissing, Normalize]
splits = RandomSplitter()(range_of(df_main))
y_names=["salary", "male", "white"]
%time to = TabularPandas(df_main, procs, cat_names, cont_names, y_names=y_names, y_block=MultiCategoryBlock(encoded=True, vocab=y_names), splits=splits)
CPU times: user 76.5 ms, sys: 1.74 ms, total: 78.2 ms
Wall time: 79.6 ms
dls = to.dataloaders()
dls.valid.show_batch()
workclass education marital-status occupation relationship race education-num_na age fnlwgt education-num salary male white
0 State-gov Some-college Divorced Other-service Unmarried Black False 36.000000 223020.001041 10.0 False False False
1 Private HS-grad Never-married Machine-op-inspct Unmarried White False 21.000001 227985.998943 9.0 False False True
2 Private Bachelors Married-civ-spouse Adm-clerical Husband White False 38.000000 297449.001415 13.0 True True True
3 Private Assoc-acdm Married-civ-spouse Craft-repair Husband White False 32.000000 185027.000104 12.0 True True True
4 Self-emp-not-inc HS-grad Never-married Craft-repair Not-in-family White False 29.000000 241430.999848 9.0 False True True
5 Local-gov Masters Never-married Prof-specialty Not-in-family White False 45.000000 33798.003413 14.0 False False True
6 Private HS-grad Married-civ-spouse Farming-fishing Husband White False 26.000000 73689.005033 9.0 False True True
7 ? HS-grad Married-civ-spouse ? Wife White False 19.000000 204868.000170 9.0 False False True
8 Federal-gov Bachelors Married-civ-spouse Prof-specialty Husband White False 43.000000 369467.994072 13.0 True True True
9 Private Assoc-voc Never-married Sales Not-in-family White False 26.000000 187576.999826 11.0 False True True

Not one-hot encoded

def _mock_multi_label(df):
    targ = []
    for row in df.itertuples():
        labels = []
        if row.salary == '>=50k': labels.append('>50k')
        if row.sex == ' Male':   labels.append('male')
        if row.race == ' White': labels.append('white')
        targ.append(' '.join(labels))
    df['target'] = np.array(targ)
    return df
path = untar_data(URLs.ADULT_SAMPLE)
df = pd.read_csv(path/'adult.csv')
df_main,df_test = df.iloc[:10000].copy(),df.iloc[10000:].copy()
df_main = _mock_multi_label(df_main)
df_main.head()
age workclass fnlwgt education education-num marital-status occupation relationship race sex capital-gain capital-loss hours-per-week native-country salary target
0 49 Private 101320 Assoc-acdm 12.0 Married-civ-spouse NaN Wife White Female 0 1902 40 United-States >=50k >50k white
1 44 Private 236746 Masters 14.0 Divorced Exec-managerial Not-in-family White Male 10520 0 45 United-States >=50k >50k male white
2 38 Private 96185 HS-grad NaN Divorced NaN Unmarried Black Female 0 0 32 United-States <50k
3 38 Self-emp-inc 112847 Prof-school 15.0 Married-civ-spouse Prof-specialty Husband Asian-Pac-Islander Male 0 0 40 United-States >=50k >50k male
4 42 Self-emp-not-inc 82297 7th-8th NaN Married-civ-spouse Other-service Wife Black Female 0 0 50 United-States <50k
@MultiCategorize
def encodes(self, to:Tabular): 
    #to.transform(to.y_names, partial(_apply_cats, {n: self.vocab for n in to.y_names}, 0))
    return to
  
@MultiCategorize
def decodes(self, to:Tabular): 
    #to.transform(to.y_names, partial(_decode_cats, {n: self.vocab for n in to.y_names}))
    return to
cat_names = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race']
cont_names = ['age', 'fnlwgt', 'education-num']
procs = [Categorify, FillMissing, Normalize]
splits = RandomSplitter()(range_of(df_main))
%time to = TabularPandas(df_main, procs, cat_names, cont_names, y_names="target", y_block=MultiCategoryBlock(), splits=splits)
CPU times: user 71.4 ms, sys: 908 ┬Ás, total: 72.3 ms
Wall time: 76.9 ms
to.procs[2].vocab
(#24) ['-','_','a','c','d','e','f','g','h','i'...]

Regression

setups[source]

setups(to:Tabular)

encodes[source]

encodes(to:Tabular)

path = untar_data(URLs.ADULT_SAMPLE)
df = pd.read_csv(path/'adult.csv')
df_main,df_test = df.iloc[:10000].copy(),df.iloc[10000:].copy()
df_main = _mock_multi_label(df_main)
cat_names = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race']
cont_names = ['fnlwgt', 'education-num']
procs = [Categorify, FillMissing, Normalize]
splits = RandomSplitter()(range_of(df_main))
%time to = TabularPandas(df_main, procs, cat_names, cont_names, y_names='age', splits=splits)
CPU times: user 70.2 ms, sys: 1.47 ms, total: 71.7 ms
Wall time: 73.7 ms
to.procs[-1].means
fnlwgt           192960.187250
education-num        10.077125
dtype: float64
dls = to.dataloaders()
dls.valid.show_batch()
workclass education marital-status occupation relationship race education-num_na fnlwgt education-num age
0 Private Some-college Divorced #na# Unmarried White True 70092.004456 10.0 41.0
1 Private Some-college Married-civ-spouse Craft-repair Husband White False 128143.001298 10.0 51.0
2 Private Some-college Divorced Adm-clerical Unmarried White False 201454.000364 10.0 38.0
3 Private HS-grad Never-married Machine-op-inspct Own-child White False 160300.000984 9.0 26.0
4 Private Bachelors Never-married Sales Not-in-family White False 176683.000798 13.0 29.0
5 Private Some-college Married-civ-spouse Machine-op-inspct Husband White False 247444.001407 10.0 30.0
6 Private 10th Separated Adm-clerical Unmarried White False 150600.998845 6.0 38.0
7 Private HS-grad Divorced Machine-op-inspct Unmarried White False 265880.996902 9.0 36.0
8 Private Assoc-voc Divorced #na# Unmarried White True 142411.000408 10.0 53.0
9 Private 11th Never-married Other-service Own-child White False 115550.999867 7.0 17.0

Not being used now - for multi-modal

class TensorTabular(Tuple):
    def get_ctxs(self, max_n=10, **kwargs):
        n_samples = min(self[0].shape[0], max_n)
        df = pd.DataFrame(index = range(n_samples))
        return [df.iloc[i] for i in range(n_samples)]

    def display(self, ctxs): display_df(pd.DataFrame(ctxs))

class TabularLine(pd.Series):
    "A line of a dataframe that knows how to show itself"
    def show(self, ctx=None, **kwargs): return self if ctx is None else ctx.append(self)

class ReadTabLine(ItemTransform):
    def __init__(self, proc): self.proc = proc

    def encodes(self, row):
        cats,conts = (o.map(row.__getitem__) for o in (self.proc.cat_names,self.proc.cont_names))
        return TensorTabular(tensor(cats).long(),tensor(conts).float())

    def decodes(self, o):
        to = TabularPandas(o, self.proc.cat_names, self.proc.cont_names, self.proc.y_names)
        to = self.proc.decode(to)
        return TabularLine(pd.Series({c: v for v,c in zip(to.items[0]+to.items[1], self.proc.cat_names+self.proc.cont_names)}))

class ReadTabTarget(ItemTransform):
    def __init__(self, proc): self.proc = proc
    def encodes(self, row): return row[self.proc.y_names].astype(np.int64)
    def decodes(self, o): return Category(self.proc.classes[self.proc.y_names][o])
# tds = TfmdDS(to.items, tfms=[[ReadTabLine(proc)], ReadTabTarget(proc)])
# enc = tds[1]
# test_eq(enc[0][0], tensor([2,1]))
# test_close(enc[0][1], tensor([-0.628828]))
# test_eq(enc[1], 1)

# dec = tds.decode(enc)
# assert isinstance(dec[0], TabularLine)
# test_close(dec[0], pd.Series({'a': 1, 'b_na': False, 'b': 1}))
# test_eq(dec[1], 'a')

# test_stdout(lambda: print(show_at(tds, 1)), """a               1
# b_na        False
# b               1
# category        a
# dtype: object""")